2024-11-18 03:48:10,169 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 03:48:10,189 main DEBUG Took 0.011679 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-18 03:48:10,189 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-18 03:48:10,190 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-18 03:48:10,190 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-18 03:48:10,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,199 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-18 03:48:10,215 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,217 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,218 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,218 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,219 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,219 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,221 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,221 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,222 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,222 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,223 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,224 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,224 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,226 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,226 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,227 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,227 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,228 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,228 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,229 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,229 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,230 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,230 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-18 03:48:10,231 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,231 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-18 03:48:10,233 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-18 03:48:10,235 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-18 03:48:10,237 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-18 03:48:10,238 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-18 03:48:10,240 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-18 03:48:10,240 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-18 03:48:10,252 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-18 03:48:10,256 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-18 03:48:10,258 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-18 03:48:10,258 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-18 03:48:10,259 main DEBUG createAppenders(={Console}) 2024-11-18 03:48:10,260 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-18 03:48:10,260 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-18 03:48:10,261 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-18 03:48:10,261 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-18 03:48:10,262 main DEBUG OutputStream closed 2024-11-18 03:48:10,262 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-18 03:48:10,263 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-18 03:48:10,263 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-18 03:48:10,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-18 03:48:10,344 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-18 03:48:10,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-18 03:48:10,345 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-18 03:48:10,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-18 03:48:10,346 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-18 03:48:10,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-18 03:48:10,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-18 03:48:10,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-18 03:48:10,347 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-18 03:48:10,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-18 03:48:10,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-18 03:48:10,348 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-18 03:48:10,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-18 03:48:10,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-18 03:48:10,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-18 03:48:10,349 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-18 03:48:10,350 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-18 03:48:10,352 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18 03:48:10,353 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-18 03:48:10,353 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-18 03:48:10,354 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-18T03:48:10,370 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-18 03:48:10,373 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-18 03:48:10,373 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-18T03:48:10,613 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd 2024-11-18T03:48:10,637 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=665, ProcessCount=11, AvailableMemoryMB=5849 2024-11-18T03:48:10,741 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-18T03:48:10,830 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:10,845 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/cluster_61e098d8-a75e-f9b0-7296-e21aeef6037d, deleteOnExit=true 2024-11-18T03:48:10,940 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/cluster_61e098d8-a75e-f9b0-7296-e21aeef6037d/zookeeper_0, clientPort=62657, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/cluster_61e098d8-a75e-f9b0-7296-e21aeef6037d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/cluster_61e098d8-a75e-f9b0-7296-e21aeef6037d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T03:48:10,954 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=62657 2024-11-18T03:48:10,955 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:11,047 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a with version=8 2024-11-18T03:48:11,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/hbase-staging 2024-11-18T03:48:11,153 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-18T03:48:11,439 INFO [Time-limited test {}] client.ConnectionUtils(128): master/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:11,449 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:11,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:11,458 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:11,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:11,458 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:11,594 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T03:48:11,687 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:48:11,725 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-18T03:48:11,735 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-18T03:48:11,740 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:11,783 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20835 (auto-detected) 2024-11-18T03:48:11,784 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-18T03:48:11,810 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34237 2024-11-18T03:48:11,839 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34237 connecting to ZooKeeper ensemble=127.0.0.1:62657 2024-11-18T03:48:11,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:342370x0, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:11,885 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34237-0x10129307eb40000 connected 2024-11-18T03:48:11,917 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:11,919 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:11,928 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:11,932 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a, hbase.cluster.distributed=false 2024-11-18T03:48:11,961 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:11,967 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34237 2024-11-18T03:48:11,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34237 2024-11-18T03:48:11,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34237 2024-11-18T03:48:11,972 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34237 2024-11-18T03:48:11,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34237 2024-11-18T03:48:12,067 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:12,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:12,070 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:12,071 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:12,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:12,071 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:12,075 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T03:48:12,077 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:12,079 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35609 2024-11-18T03:48:12,080 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35609 connecting to ZooKeeper ensemble=127.0.0.1:62657 2024-11-18T03:48:12,081 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:12,082 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:12,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:356090x0, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:12,086 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:356090x0, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:12,086 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35609-0x10129307eb40001 connected 2024-11-18T03:48:12,092 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T03:48:12,107 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.cache 2024-11-18T03:48:12,115 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T03:48:12,117 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T03:48:12,119 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T03:48:12,121 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T03:48:12,121 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T03:48:12,128 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T03:48:12,129 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T03:48:12,129 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.persistence is complete. 2024-11-18T03:48:12,134 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T03:48:12,137 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T03:48:12,142 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:12,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35609 2024-11-18T03:48:12,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35609 2024-11-18T03:48:12,145 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35609 2024-11-18T03:48:12,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35609 2024-11-18T03:48:12,149 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35609 2024-11-18T03:48:12,168 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:12,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:12,168 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:12,169 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:12,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:12,169 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:12,169 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T03:48:12,170 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:12,171 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36815 2024-11-18T03:48:12,172 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36815 connecting to ZooKeeper ensemble=127.0.0.1:62657 2024-11-18T03:48:12,173 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:12,174 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:12,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368150x0, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:12,180 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:368150x0, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:12,180 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36815-0x10129307eb40002 connected 2024-11-18T03:48:12,181 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T03:48:12,184 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.cache 2024-11-18T03:48:12,185 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T03:48:12,185 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T03:48:12,185 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T03:48:12,186 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T03:48:12,186 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T03:48:12,186 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T03:48:12,187 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T03:48:12,187 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T03:48:12,187 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.persistence is complete. 2024-11-18T03:48:12,188 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T03:48:12,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:12,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36815 2024-11-18T03:48:12,199 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36815 2024-11-18T03:48:12,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36815 2024-11-18T03:48:12,204 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36815 2024-11-18T03:48:12,205 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36815 2024-11-18T03:48:12,229 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;88c1dcece5ca:34237 2024-11-18T03:48:12,230 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:12,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:12,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:12,241 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:12,244 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:12,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:12,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:12,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,279 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T03:48:12,281 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/88c1dcece5ca,34237,1731901691239 from backup master directory 2024-11-18T03:48:12,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:12,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:12,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:12,284 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:12,285 WARN [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:12,285 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:12,288 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-18T03:48:12,291 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-18T03:48:12,299 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/hbase.id] with ID: 88e4c730-eadd-42fc-9098-e7fc43dd3ccf 2024-11-18T03:48:12,299 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.tmp/hbase.id 2024-11-18T03:48:12,305 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/hbase.id] 2024-11-18T03:48:12,347 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:12,353 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T03:48:12,374 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-18T03:48:12,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,379 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,412 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:48:12,417 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T03:48:12,470 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:48:12,478 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:12,541 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store 2024-11-18T03:48:12,568 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-18T03:48:12,572 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:12,574 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:48:12,575 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:12,575 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:12,577 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:48:12,578 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:12,578 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:12,579 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731901692574Disabling compacts and flushes for region at 1731901692574Disabling writes for close at 1731901692578 (+4 ms)Writing region close event to WAL at 1731901692578Closed at 1731901692578 2024-11-18T03:48:12,579 WARN [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/.initializing 2024-11-18T03:48:12,580 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/WALs/88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:12,595 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:12,609 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88c1dcece5ca%2C34237%2C1731901691239, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/WALs/88c1dcece5ca,34237,1731901691239, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/oldWALs, maxLogs=10 2024-11-18T03:48:12,650 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/WALs/88c1dcece5ca,34237,1731901691239/88c1dcece5ca%2C34237%2C1731901691239.1731901692614 2024-11-18T03:48:12,651 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:12,651 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:12,652 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:12,655 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,656 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,732 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T03:48:12,737 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:12,739 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:12,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T03:48:12,751 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:12,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:12,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T03:48:12,758 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:12,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:12,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,764 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T03:48:12,764 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:12,765 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:12,766 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,769 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,770 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,773 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,773 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,776 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:12,778 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:12,786 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:12,787 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70208284, jitterRate=0.0461849570274353}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:12,794 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731901692669Initializing all the Stores at 1731901692670 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901692670Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901692671 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901692672 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901692672Cleaning up temporary data from old regions at 1731901692773 (+101 ms)Region opened successfully at 1731901692794 (+21 ms) 2024-11-18T03:48:12,795 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T03:48:12,828 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10f6950f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:12,862 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T03:48:12,876 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T03:48:12,876 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T03:48:12,879 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T03:48:12,880 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T03:48:12,886 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-18T03:48:12,886 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T03:48:12,912 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T03:48:12,922 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T03:48:12,925 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T03:48:12,927 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T03:48:12,929 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T03:48:12,931 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T03:48:12,933 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T03:48:12,937 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T03:48:12,939 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T03:48:12,941 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T03:48:12,942 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T03:48:12,969 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T03:48:12,972 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T03:48:12,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:12,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:12,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:12,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:12,991 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=88c1dcece5ca,34237,1731901691239, sessionid=0x10129307eb40000, setting cluster-up flag (Was=false) 2024-11-18T03:48:13,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,014 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T03:48:13,017 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:13,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,032 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T03:48:13,034 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:13,040 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T03:48:13,110 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(746): ClusterId : 88e4c730-eadd-42fc-9098-e7fc43dd3ccf 2024-11-18T03:48:13,110 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(746): ClusterId : 88e4c730-eadd-42fc-9098-e7fc43dd3ccf 2024-11-18T03:48:13,114 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T03:48:13,114 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T03:48:13,121 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T03:48:13,121 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T03:48:13,121 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T03:48:13,122 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T03:48:13,125 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T03:48:13,125 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T03:48:13,126 DEBUG [RS:0;88c1dcece5ca:35609 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@187f8c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:13,126 DEBUG [RS:1;88c1dcece5ca:36815 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@457c8c1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:13,126 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:13,141 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T03:48:13,145 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;88c1dcece5ca:35609 2024-11-18T03:48:13,151 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T03:48:13,151 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T03:48:13,152 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T03:48:13,152 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T03:48:13,152 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;88c1dcece5ca:36815 2024-11-18T03:48:13,152 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T03:48:13,152 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T03:48:13,152 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T03:48:13,156 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(2659): reportForDuty to master=88c1dcece5ca,34237,1731901691239 with port=36815, startcode=1731901692167 2024-11-18T03:48:13,156 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(2659): reportForDuty to master=88c1dcece5ca,34237,1731901691239 with port=35609, startcode=1731901692024 2024-11-18T03:48:13,159 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 88c1dcece5ca,34237,1731901691239 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T03:48:13,168 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:13,169 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:13,170 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:13,170 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:13,170 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/88c1dcece5ca:0, corePoolSize=10, maxPoolSize=10 2024-11-18T03:48:13,171 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,171 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:13,172 DEBUG [RS:0;88c1dcece5ca:35609 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T03:48:13,172 DEBUG [RS:1;88c1dcece5ca:36815 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T03:48:13,172 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,177 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:13,177 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T03:48:13,182 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:13,183 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T03:48:13,185 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731901723184 2024-11-18T03:48:13,187 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T03:48:13,188 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T03:48:13,192 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T03:48:13,192 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T03:48:13,193 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T03:48:13,193 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T03:48:13,195 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,205 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T03:48:13,207 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T03:48:13,207 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T03:48:13,215 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T03:48:13,215 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T03:48:13,218 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901693216,5,FailOnTimeoutGroup] 2024-11-18T03:48:13,219 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T03:48:13,219 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901693218,5,FailOnTimeoutGroup] 2024-11-18T03:48:13,219 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,219 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:13,220 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T03:48:13,221 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,223 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:13,245 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T03:48:13,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:48:13,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:13,254 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:13,255 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T03:48:13,264 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:48:13,264 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:13,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:13,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T03:48:13,265 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56287, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T03:48:13,265 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49001, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T03:48:13,272 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34237 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:13,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:48:13,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:13,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:13,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T03:48:13,275 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34237 {}] master.ServerManager(517): Registering regionserver=88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:13,281 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:48:13,281 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:13,281 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:13,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:48:13,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:13,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:13,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:48:13,284 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:48:13,284 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:13,285 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:48:13,286 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34237 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:13,287 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34237 {}] master.ServerManager(517): Registering regionserver=88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:13,292 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:13,292 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T03:48:13,292 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T03:48:13,293 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:13,293 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T03:48:13,293 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T03:48:13,302 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:13,302 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64312171, jitterRate=-0.04167397320270538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:13,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:13,305 DEBUG [RS:1;88c1dcece5ca:36815 {}] zookeeper.ZKUtil(111): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:13,305 WARN [RS:1;88c1dcece5ca:36815 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:13,305 DEBUG [RS:0;88c1dcece5ca:35609 {}] zookeeper.ZKUtil(111): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:13,305 INFO [RS:1;88c1dcece5ca:36815 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:13,305 WARN [RS:0;88c1dcece5ca:35609 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:13,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731901693240Initializing all the Stores at 1731901693240Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901693240Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901693244 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901693244Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901693244Cleaning up temporary data from old regions at 1731901693284 (+40 ms)Region opened successfully at 1731901693305 (+21 ms) 2024-11-18T03:48:13,306 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:13,306 INFO [RS:0;88c1dcece5ca:35609 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:13,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:48:13,306 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:48:13,306 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:13,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:48:13,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:48:13,306 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:48:13,310 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88c1dcece5ca,35609,1731901692024] 2024-11-18T03:48:13,310 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88c1dcece5ca,36815,1731901692167] 2024-11-18T03:48:13,312 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:13,312 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731901693306Disabling compacts and flushes for region at 1731901693306Disabling writes for close at 1731901693306Writing region close event to WAL at 1731901693311 (+5 ms)Closed at 1731901693312 (+1 ms) 2024-11-18T03:48:13,314 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:13,314 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T03:48:13,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T03:48:13,333 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:48:13,336 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T03:48:13,342 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T03:48:13,342 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T03:48:13,356 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T03:48:13,356 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T03:48:13,362 INFO [RS:1;88c1dcece5ca:36815 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T03:48:13,362 INFO [RS:0;88c1dcece5ca:35609 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T03:48:13,362 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,362 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,363 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T03:48:13,364 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T03:48:13,371 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T03:48:13,371 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T03:48:13,372 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,373 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,373 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,373 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,373 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,373 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,373 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,373 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,373 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:13,374 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:13,374 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,374 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,375 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,375 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,375 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,375 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,375 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,375 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,375 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:13,375 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:13,376 DEBUG [RS:0;88c1dcece5ca:35609 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:13,376 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:13,376 DEBUG [RS:1;88c1dcece5ca:36815 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:13,381 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,381 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,381 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,36815,1731901692167-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,382 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,383 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,35609,1731901692024-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:13,406 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T03:48:13,406 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T03:48:13,408 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,35609,1731901692024-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,408 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,36815,1731901692167-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,409 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,409 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,409 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.Replication(171): 88c1dcece5ca,36815,1731901692167 started 2024-11-18T03:48:13,409 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.Replication(171): 88c1dcece5ca,35609,1731901692024 started 2024-11-18T03:48:13,434 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,435 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1482): Serving as 88c1dcece5ca,35609,1731901692024, RpcServer on 88c1dcece5ca/172.17.0.2:35609, sessionid=0x10129307eb40001 2024-11-18T03:48:13,436 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T03:48:13,437 DEBUG [RS:0;88c1dcece5ca:35609 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:13,437 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,35609,1731901692024' 2024-11-18T03:48:13,437 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:13,437 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T03:48:13,437 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1482): Serving as 88c1dcece5ca,36815,1731901692167, RpcServer on 88c1dcece5ca/172.17.0.2:36815, sessionid=0x10129307eb40002 2024-11-18T03:48:13,438 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T03:48:13,438 DEBUG [RS:1;88c1dcece5ca:36815 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:13,438 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,36815,1731901692167' 2024-11-18T03:48:13,438 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T03:48:13,438 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T03:48:13,439 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T03:48:13,440 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T03:48:13,440 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T03:48:13,440 DEBUG [RS:0;88c1dcece5ca:35609 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:13,440 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,35609,1731901692024' 2024-11-18T03:48:13,440 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T03:48:13,440 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T03:48:13,440 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T03:48:13,440 DEBUG [RS:1;88c1dcece5ca:36815 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:13,440 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,36815,1731901692167' 2024-11-18T03:48:13,441 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T03:48:13,441 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T03:48:13,441 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T03:48:13,442 DEBUG [RS:0;88c1dcece5ca:35609 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T03:48:13,442 DEBUG [RS:1;88c1dcece5ca:36815 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T03:48:13,443 INFO [RS:0;88c1dcece5ca:35609 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T03:48:13,443 INFO [RS:1;88c1dcece5ca:36815 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T03:48:13,443 INFO [RS:0;88c1dcece5ca:35609 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T03:48:13,443 INFO [RS:1;88c1dcece5ca:36815 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T03:48:13,487 WARN [88c1dcece5ca:34237 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T03:48:13,548 INFO [RS:1;88c1dcece5ca:36815 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:13,548 INFO [RS:0;88c1dcece5ca:35609 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:13,549 INFO [RS:0;88c1dcece5ca:35609 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C35609%2C1731901692024, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,35609,1731901692024, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs, maxLogs=55 2024-11-18T03:48:13,549 INFO [RS:1;88c1dcece5ca:36815 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C36815%2C1731901692167, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,36815,1731901692167, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs, maxLogs=55 2024-11-18T03:48:13,576 INFO [RS:0;88c1dcece5ca:35609 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,35609,1731901692024/88c1dcece5ca%2C35609%2C1731901692024.1731901693554 2024-11-18T03:48:13,576 DEBUG [RS:0;88c1dcece5ca:35609 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:13,578 INFO [RS:1;88c1dcece5ca:36815 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,36815,1731901692167/88c1dcece5ca%2C36815%2C1731901692167.1731901693554 2024-11-18T03:48:13,579 DEBUG [RS:1;88c1dcece5ca:36815 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:13,740 DEBUG [88c1dcece5ca:34237 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T03:48:13,754 DEBUG [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(204): Hosts are {88c1dcece5ca=0} racks are {/default-rack=0} 2024-11-18T03:48:13,757 DEBUG [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T03:48:13,757 DEBUG [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T03:48:13,757 DEBUG [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T03:48:13,757 DEBUG [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T03:48:13,757 INFO [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T03:48:13,757 INFO [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T03:48:13,757 DEBUG [88c1dcece5ca:34237 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T03:48:13,764 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:13,771 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88c1dcece5ca,36815,1731901692167, state=OPENING 2024-11-18T03:48:13,776 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T03:48:13,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:13,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:13,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:13,779 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:13,781 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:48:13,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=88c1dcece5ca,36815,1731901692167}] 2024-11-18T03:48:13,956 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T03:48:13,960 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38867, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T03:48:13,969 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T03:48:13,969 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:13,970 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T03:48:13,971 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C36815%2C1731901692167.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,36815,1731901692167, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs, maxLogs=55 2024-11-18T03:48:13,991 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,36815,1731901692167/88c1dcece5ca%2C36815%2C1731901692167.meta.1731901693973.meta 2024-11-18T03:48:13,992 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:13,992 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:13,994 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T03:48:13,996 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T03:48:14,000 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T03:48:14,004 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T03:48:14,005 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:14,005 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T03:48:14,005 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T03:48:14,007 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T03:48:14,008 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:48:14,008 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:14,008 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:14,009 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T03:48:14,010 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:48:14,010 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:14,010 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:14,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T03:48:14,012 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:48:14,012 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:14,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:14,012 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T03:48:14,013 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:48:14,013 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:14,014 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:14,014 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:48:14,015 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:14,016 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:14,016 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:48:14,016 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:48:14,016 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:14,017 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:48:14,018 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60104035, jitterRate=-0.10438008606433868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:14,018 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T03:48:14,020 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731901694006Writing region info on filesystem at 1731901694006Initializing all the Stores at 1731901694006Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901694006Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901694007 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901694007Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901694007Cleaning up temporary data from old regions at 1731901694016 (+9 ms)Running coprocessor post-open hooks at 1731901694018 (+2 ms)Region opened successfully at 1731901694020 (+2 ms) 2024-11-18T03:48:14,026 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731901693948 2024-11-18T03:48:14,036 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T03:48:14,037 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T03:48:14,038 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:14,040 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88c1dcece5ca,36815,1731901692167, state=OPEN 2024-11-18T03:48:14,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:14,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:14,042 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:14,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:14,042 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:14,043 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:14,043 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:14,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T03:48:14,046 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88c1dcece5ca,36815,1731901692167 in 261 msec 2024-11-18T03:48:14,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T03:48:14,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 724 msec 2024-11-18T03:48:14,051 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:14,051 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T03:48:14,072 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:48:14,074 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88c1dcece5ca,36815,1731901692167, seqNum=-1] 2024-11-18T03:48:14,095 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:48:14,097 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40987, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:48:14,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0520 sec 2024-11-18T03:48:14,123 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731901694123, completionTime=-1 2024-11-18T03:48:14,126 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T03:48:14,126 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T03:48:14,150 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T03:48:14,150 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731901754150 2024-11-18T03:48:14,150 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731901814150 2024-11-18T03:48:14,151 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-18T03:48:14,153 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-18T03:48:14,159 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,34237,1731901691239-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:14,159 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,34237,1731901691239-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:14,159 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,34237,1731901691239-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:14,161 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-88c1dcece5ca:34237, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:14,162 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:14,162 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:14,168 DEBUG [master/88c1dcece5ca:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T03:48:14,193 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.907sec 2024-11-18T03:48:14,195 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T03:48:14,196 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T03:48:14,198 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T03:48:14,198 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T03:48:14,198 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T03:48:14,199 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,34237,1731901691239-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:14,200 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,34237,1731901691239-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T03:48:14,201 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T03:48:14,202 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T03:48:14,202 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,34237,1731901691239-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:14,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:14,222 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-18T03:48:14,222 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-18T03:48:14,226 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 88c1dcece5ca,34237,-1 for getting cluster id 2024-11-18T03:48:14,229 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:48:14,237 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '88e4c730-eadd-42fc-9098-e7fc43dd3ccf' 2024-11-18T03:48:14,239 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:48:14,239 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "88e4c730-eadd-42fc-9098-e7fc43dd3ccf" 2024-11-18T03:48:14,240 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48fae47b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:14,240 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [88c1dcece5ca,34237,-1] 2024-11-18T03:48:14,242 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:48:14,244 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:14,245 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:48:14,248 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:14,248 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:48:14,255 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88c1dcece5ca,36815,1731901692167, seqNum=-1] 2024-11-18T03:48:14,256 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:48:14,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:48:14,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:14,282 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:48:14,286 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:14,289 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@245f772f 2024-11-18T03:48:14,290 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:48:14,292 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:48:14,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731901694278', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:48:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 2024-11-18T03:48:14,316 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:48:14,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731901694278" procId is: 4 2024-11-18T03:48:14,321 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:14,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:48:14,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:14,339 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f6e69dbfeb16d794a2d889fdb08a52d6, NAME => 'testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731901694278', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:14,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:14,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(1722): Closing f6e69dbfeb16d794a2d889fdb08a52d6, disabling compactions & flushes 2024-11-18T03:48:14,347 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:14,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:14,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. after waiting 0 ms 2024-11-18T03:48:14,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:14,347 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:14,347 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731901694278-pool-0 {}] regionserver.HRegion(1676): Region close journal for f6e69dbfeb16d794a2d889fdb08a52d6: Waiting for close lock at 1731901694347Disabling compacts and flushes for region at 1731901694347Disabling writes for close at 1731901694347Writing region close event to WAL at 1731901694347Closed at 1731901694347 2024-11-18T03:48:14,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:48:14,353 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731901694349"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731901694349"}]},"ts":"1731901694349"} 2024-11-18T03:48:14,358 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:48:14,359 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:48:14,361 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731901694278","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731901694359"}]},"ts":"1731901694359"} 2024-11-18T03:48:14,365 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731901694278, state=ENABLING in hbase:meta 2024-11-18T03:48:14,365 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {88c1dcece5ca=0} racks are {/default-rack=0} 2024-11-18T03:48:14,366 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T03:48:14,366 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T03:48:14,366 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T03:48:14,366 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T03:48:14,366 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T03:48:14,366 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T03:48:14,366 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T03:48:14,367 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, ASSIGN}] 2024-11-18T03:48:14,369 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, ASSIGN 2024-11-18T03:48:14,370 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, ASSIGN; state=OFFLINE, location=88c1dcece5ca,36815,1731901692167; forceNewPlan=false, retain=false 2024-11-18T03:48:14,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:14,522 INFO [88c1dcece5ca:34237 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T03:48:14,523 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f6e69dbfeb16d794a2d889fdb08a52d6, regionState=OPENING, regionLocation=88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:14,527 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, ASSIGN because future has completed 2024-11-18T03:48:14,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,36815,1731901692167}] 2024-11-18T03:48:14,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:14,684 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:14,684 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f6e69dbfeb16d794a2d889fdb08a52d6, NAME => 'testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:14,685 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731901694278 f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,685 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:14,685 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,685 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,686 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,692 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6e69dbfeb16d794a2d889fdb08a52d6 columnFamilyName family 2024-11-18T03:48:14,692 DEBUG [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:14,693 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.HStore(327): Store=f6e69dbfeb16d794a2d889fdb08a52d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:14,693 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,694 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,694 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,694 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,694 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,695 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,702 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:14,703 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f6e69dbfeb16d794a2d889fdb08a52d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67375948, jitterRate=0.003979861736297607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T03:48:14,703 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:14,704 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f6e69dbfeb16d794a2d889fdb08a52d6: Running coprocessor pre-open hook at 1731901694686Writing region info on filesystem at 1731901694686Initializing all the Stores at 1731901694686Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901694686Cleaning up temporary data from old regions at 1731901694694 (+8 ms)Running coprocessor post-open hooks at 1731901694703 (+9 ms)Region opened successfully at 1731901694704 (+1 ms) 2024-11-18T03:48:14,705 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6., pid=6, masterSystemTime=1731901694681 2024-11-18T03:48:14,709 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:14,709 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:14,709 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f6e69dbfeb16d794a2d889fdb08a52d6, regionState=OPEN, openSeqNum=2, regionLocation=88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:14,713 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,36815,1731901692167 because future has completed 2024-11-18T03:48:14,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T03:48:14,716 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,36815,1731901692167 in 186 msec 2024-11-18T03:48:14,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T03:48:14,719 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, ASSIGN in 349 msec 2024-11-18T03:48:14,720 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:48:14,720 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731901694278","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731901694720"}]},"ts":"1731901694720"} 2024-11-18T03:48:14,723 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731901694278, state=ENABLED in hbase:meta 2024-11-18T03:48:14,723 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:48:14,725 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731901694278 in 412 msec 2024-11-18T03:48:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:14,963 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731901694278 completed 2024-11-18T03:48:14,963 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731901694278 get assigned. Timeout = 60000ms 2024-11-18T03:48:14,966 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T03:48:14,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731901694278 assigned to meta. Checking AM states. 2024-11-18T03:48:14,972 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T03:48:14,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731901694278 assigned. 2024-11-18T03:48:14,980 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731901694278', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6., hostname=88c1dcece5ca,36815,1731901692167, seqNum=2] 2024-11-18T03:48:14,990 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f6e69dbfeb16d794a2d889fdb08a52d6 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T03:48:15,017 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:15,057 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/.tmp/family/b066f75a94114671a2710afd1bb41143 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T03:48:15,062 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/.tmp/family/b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:15,098 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/.tmp/family/b066f75a94114671a2710afd1bb41143 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:15,099 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143, delay=933 ms 2024-11-18T03:48:15,101 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T03:48:15,108 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f6e69dbfeb16d794a2d889fdb08a52d6 in 115ms, sequenceid=6, compaction requested=false 2024-11-18T03:48:15,109 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731901694278' 2024-11-18T03:48:15,111 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f6e69dbfeb16d794a2d889fdb08a52d6: 2024-11-18T03:48:16,035 DEBUG [hfile-prefetch-1731901695100 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b066f75a94114671a2710afd1bb41143_0. 2024-11-18T03:48:16,039 DEBUG [hfile-prefetch-1731901695100 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key b066f75a94114671a2710afd1bb41143_117. 2024-11-18T03:48:16,040 DEBUG [hfile-prefetch-1731901695100 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143, with total blocks 2, and data blocks 1 2024-11-18T03:48:16,040 DEBUG [hfile-prefetch-1731901695100 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T03:48:16,041 DEBUG [hfile-prefetch-1731901695100 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T03:48:16,041 DEBUG [hfile-prefetch-1731901695100 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143. Will sleep 100ms and try the verification again. 2024-11-18T03:48:16,089 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T03:48:16,089 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T03:48:16,090 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:16,090 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:16,109 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T03:48:16,109 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731901696089 2024-11-18T03:48:16,109 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T03:48:16,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=f6e69dbfeb16d794a2d889fdb08a52d6, source=88c1dcece5ca,36815,1731901692167, destination=88c1dcece5ca,35609,1731901692024, warming up region on 88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:16,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T03:48:16,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=f6e69dbfeb16d794a2d889fdb08a52d6, source=88c1dcece5ca,36815,1731901692167, destination=88c1dcece5ca,35609,1731901692024, running balancer 2024-11-18T03:48:16,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, REOPEN/MOVE 2024-11-18T03:48:16,124 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59341, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T03:48:16,124 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, REOPEN/MOVE 2024-11-18T03:48:16,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(7855): Warmup {ENCODED => f6e69dbfeb16d794a2d889fdb08a52d6, NAME => 'testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:16,126 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f6e69dbfeb16d794a2d889fdb08a52d6, regionState=CLOSING, regionLocation=88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:16,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:16,127 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,128 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6e69dbfeb16d794a2d889fdb08a52d6 columnFamilyName family 2024-11-18T03:48:16,128 DEBUG [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:16,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, REOPEN/MOVE because future has completed 2024-11-18T03:48:16,129 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-18T03:48:16,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,36815,1731901692167}] 2024-11-18T03:48:16,133 DEBUG [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:16,133 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.HStore(327): Store=f6e69dbfeb16d794a2d889fdb08a52d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(1722): Closing f6e69dbfeb16d794a2d889fdb08a52d6, disabling compactions & flushes 2024-11-18T03:48:16,133 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. after waiting 0 ms 2024-11-18T03:48:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,136 DEBUG [StoreFileCloser-f6e69dbfeb16d794a2d889fdb08a52d6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:16,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35609 {}] regionserver.HRegion(1676): Region close journal for f6e69dbfeb16d794a2d889fdb08a52d6: Waiting for close lock at 1731901696133Disabling compacts and flushes for region at 1731901696133Disabling writes for close at 1731901696134 (+1 ms)Writing region close event to WAL at 1731901696137 (+3 ms)Closed at 1731901696137 2024-11-18T03:48:16,138 WARN [hfile-prefetch-1731901695100 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T03:48:16,139 DEBUG [hfile-prefetch-1731901695100 {}] hfile.PrefetchExecutor(121): Prefetch completed for b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34237 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-18T03:48:16,288 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,289 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-18T03:48:16,290 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing f6e69dbfeb16d794a2d889fdb08a52d6, disabling compactions & flushes 2024-11-18T03:48:16,290 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,290 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,290 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. after waiting 0 ms 2024-11-18T03:48:16,290 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,292 DEBUG [StoreFileCloser-f6e69dbfeb16d794a2d889fdb08a52d6-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file b066f75a94114671a2710afd1bb41143, starting offset: 0, end offset: 9223372036854775807 2024-11-18T03:48:16,294 DEBUG [StoreFileCloser-f6e69dbfeb16d794a2d889fdb08a52d6-family-1 {}] bucket.BucketCache(856): removed key b066f75a94114671a2710afd1bb41143_0 from back map with offset lock 2094080 in the evict process 2024-11-18T03:48:16,300 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T03:48:16,302 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,302 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for f6e69dbfeb16d794a2d889fdb08a52d6: Waiting for close lock at 1731901696290Running coprocessor pre-close hooks at 1731901696290Disabling compacts and flushes for region at 1731901696290Disabling writes for close at 1731901696290Writing region close event to WAL at 1731901696294 (+4 ms)Running coprocessor post-close hooks at 1731901696300 (+6 ms)Closed at 1731901696301 (+1 ms) 2024-11-18T03:48:16,302 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding f6e69dbfeb16d794a2d889fdb08a52d6 move to 88c1dcece5ca,35609,1731901692024 record at close sequenceid=6 2024-11-18T03:48:16,305 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,306 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f6e69dbfeb16d794a2d889fdb08a52d6, regionState=CLOSED 2024-11-18T03:48:16,309 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T03:48:16,309 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T03:48:16,309 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:16,309 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:16,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,36815,1731901692167 because future has completed 2024-11-18T03:48:16,313 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T03:48:16,313 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731901696309 2024-11-18T03:48:16,313 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T03:48:16,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T03:48:16,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,36815,1731901692167 in 181 msec 2024-11-18T03:48:16,314 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, REOPEN/MOVE; state=CLOSED, location=88c1dcece5ca,35609,1731901692024; forceNewPlan=false, retain=false 2024-11-18T03:48:16,465 INFO [88c1dcece5ca:34237 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T03:48:16,465 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f6e69dbfeb16d794a2d889fdb08a52d6, regionState=OPENING, regionLocation=88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:16,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, REOPEN/MOVE because future has completed 2024-11-18T03:48:16,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,35609,1731901692024}] 2024-11-18T03:48:16,625 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,625 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f6e69dbfeb16d794a2d889fdb08a52d6, NAME => 'testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:16,625 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731901694278 f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,626 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:16,626 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,626 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,627 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,628 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6e69dbfeb16d794a2d889fdb08a52d6 columnFamilyName family 2024-11-18T03:48:16,628 DEBUG [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:16,630 DEBUG [StoreFileOpener-f6e69dbfeb16d794a2d889fdb08a52d6-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143, delay=942 ms 2024-11-18T03:48:16,632 DEBUG [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:16,632 INFO [StoreOpener-f6e69dbfeb16d794a2d889fdb08a52d6-1 {}] regionserver.HStore(327): Store=f6e69dbfeb16d794a2d889fdb08a52d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:16,633 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,633 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,634 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,634 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,634 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,638 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,639 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f6e69dbfeb16d794a2d889fdb08a52d6; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75004961, jitterRate=0.1176610141992569}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T03:48:16,639 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,640 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f6e69dbfeb16d794a2d889fdb08a52d6: Running coprocessor pre-open hook at 1731901696626Writing region info on filesystem at 1731901696626Initializing all the Stores at 1731901696626Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901696626Cleaning up temporary data from old regions at 1731901696634 (+8 ms)Running coprocessor post-open hooks at 1731901696639 (+5 ms)Region opened successfully at 1731901696640 (+1 ms) 2024-11-18T03:48:16,641 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6., pid=9, masterSystemTime=1731901696622 2024-11-18T03:48:16,644 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,644 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,644 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f6e69dbfeb16d794a2d889fdb08a52d6, regionState=OPEN, openSeqNum=10, regionLocation=88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:16,648 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,35609,1731901692024 because future has completed 2024-11-18T03:48:16,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-18T03:48:16,651 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f6e69dbfeb16d794a2d889fdb08a52d6, server=88c1dcece5ca,35609,1731901692024 in 180 msec 2024-11-18T03:48:16,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731901694278, region=f6e69dbfeb16d794a2d889fdb08a52d6, REOPEN/MOVE in 528 msec 2024-11-18T03:48:16,677 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T03:48:16,678 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:48:16,678 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:16,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:16,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:16,683 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:48:16,683 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T03:48:16,683 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-18T03:48:16,684 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=88c1dcece5ca,34237,1731901691239 2024-11-18T03:48:16,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:16,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:16,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:16,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:16,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:16,686 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T03:48:16,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:16,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:16,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:16,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:16,688 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:48:16,688 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:16,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:16,688 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88c1dcece5ca,35609,1731901692024' ***** 2024-11-18T03:48:16,689 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T03:48:16,689 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88c1dcece5ca,36815,1731901692167' ***** 2024-11-18T03:48:16,689 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T03:48:16,689 INFO [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T03:48:16,689 INFO [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T03:48:16,689 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T03:48:16,689 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T03:48:16,689 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T03:48:16,689 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T03:48:16,689 INFO [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.persistence 2024-11-18T03:48:16,689 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T03:48:16,689 INFO [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/bucket.persistence 2024-11-18T03:48:16,689 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T03:48:16,690 INFO [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T03:48:16,690 INFO [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T03:48:16,690 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T03:48:16,690 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T03:48:16,700 DEBUG [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1487): Thread RS:1;88c1dcece5ca:36815 started persisting bucket cache to file 2024-11-18T03:48:16,700 DEBUG [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1487): Thread RS:0;88c1dcece5ca:35609 started persisting bucket cache to file 2024-11-18T03:48:16,700 DEBUG [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:16,700 DEBUG [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:16,700 DEBUG [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:16,700 DEBUG [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:16,703 WARN [Thread-95 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-18T03:48:16,704 DEBUG [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T03:48:16,704 DEBUG [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731901696700 2024-11-18T03:48:16,704 DEBUG [RS:0;88c1dcece5ca:35609 {}] bucket.BucketCache(1509): Thread RS:0;88c1dcece5ca:35609 finished persisting bucket cache to file, renaming 2024-11-18T03:48:16,703 ERROR [RS:1;88c1dcece5ca:36815 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-18T03:48:16,704 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T03:48:16,704 DEBUG [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T03:48:16,704 DEBUG [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731901696700 2024-11-18T03:48:16,704 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T03:48:16,704 INFO [RS:0;88c1dcece5ca:35609 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T03:48:16,704 DEBUG [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1509): Thread RS:1;88c1dcece5ca:36815 finished persisting bucket cache to file, renaming 2024-11-18T03:48:16,705 WARN [RS:1;88c1dcece5ca:36815 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-18T03:48:16,705 INFO [RS:0;88c1dcece5ca:35609 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T03:48:16,705 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T03:48:16,705 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(3091): Received CLOSE for f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,705 INFO [RS:1;88c1dcece5ca:36815 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T03:48:16,705 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T03:48:16,705 INFO [RS:1;88c1dcece5ca:36815 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T03:48:16,705 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(959): stopping server 88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:16,705 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:16,705 INFO [RS:1;88c1dcece5ca:36815 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;88c1dcece5ca:36815. 2024-11-18T03:48:16,705 DEBUG [RS:1;88c1dcece5ca:36815 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:16,705 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(959): stopping server 88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:16,705 DEBUG [RS:1;88c1dcece5ca:36815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:16,705 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:16,706 INFO [RS:0;88c1dcece5ca:35609 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;88c1dcece5ca:35609. 2024-11-18T03:48:16,706 DEBUG [RS:0;88c1dcece5ca:35609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:16,706 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T03:48:16,706 DEBUG [RS:0;88c1dcece5ca:35609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:16,706 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T03:48:16,706 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T03:48:16,706 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T03:48:16,706 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T03:48:16,706 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1325): Online Regions={f6e69dbfeb16d794a2d889fdb08a52d6=testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6.} 2024-11-18T03:48:16,706 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T03:48:16,706 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T03:48:16,706 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-18T03:48:16,706 DEBUG [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1351): Waiting on f6e69dbfeb16d794a2d889fdb08a52d6 2024-11-18T03:48:16,706 DEBUG [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-18T03:48:16,706 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f6e69dbfeb16d794a2d889fdb08a52d6, disabling compactions & flushes 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:48:16,707 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,707 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. after waiting 0 ms 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:48:16,707 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:48:16,707 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-18T03:48:16,709 DEBUG [StoreFileCloser-f6e69dbfeb16d794a2d889fdb08a52d6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/family/b066f75a94114671a2710afd1bb41143 2024-11-18T03:48:16,714 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnRegionMove1731901694278/f6e69dbfeb16d794a2d889fdb08a52d6/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T03:48:16,715 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,715 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f6e69dbfeb16d794a2d889fdb08a52d6: Waiting for close lock at 1731901696706Running coprocessor pre-close hooks at 1731901696706Disabling compacts and flushes for region at 1731901696706Disabling writes for close at 1731901696707 (+1 ms)Writing region close event to WAL at 1731901696710 (+3 ms)Running coprocessor post-close hooks at 1731901696715 (+5 ms)Closed at 1731901696715 2024-11-18T03:48:16,715 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6. 2024-11-18T03:48:16,719 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b6a4699159f44dd99b89eeb5d6148af8 2024-11-18T03:48:16,737 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/info/b6a4699159f44dd99b89eeb5d6148af8 is 205, key is testBlockEvictionOnRegionMove1731901694278,,1731901694304.f6e69dbfeb16d794a2d889fdb08a52d6./info:regioninfo/1731901696644/Put/seqid=0 2024-11-18T03:48:16,738 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/info/b6a4699159f44dd99b89eeb5d6148af8 2024-11-18T03:48:16,751 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d172d81fa3c344e99cf635265ea78472 2024-11-18T03:48:16,773 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/ns/d172d81fa3c344e99cf635265ea78472 is 43, key is default/ns:d/1731901694101/Put/seqid=0 2024-11-18T03:48:16,774 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/ns/d172d81fa3c344e99cf635265ea78472 2024-11-18T03:48:16,785 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 921868a6c61549009ac0350e4dc80e3f 2024-11-18T03:48:16,786 INFO [regionserver/88c1dcece5ca:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:16,787 INFO [regionserver/88c1dcece5ca:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:16,798 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/table/921868a6c61549009ac0350e4dc80e3f is 78, key is testBlockEvictionOnRegionMove1731901694278/table:state/1731901694720/Put/seqid=0 2024-11-18T03:48:16,799 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/table/921868a6c61549009ac0350e4dc80e3f 2024-11-18T03:48:16,801 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/info/b6a4699159f44dd99b89eeb5d6148af8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/b6a4699159f44dd99b89eeb5d6148af8 2024-11-18T03:48:16,802 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/b6a4699159f44dd99b89eeb5d6148af8, delay=902 ms 2024-11-18T03:48:16,804 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/b6a4699159f44dd99b89eeb5d6148af8, entries=15, sequenceid=15, filesize=7.6 K 2024-11-18T03:48:16,804 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/ns/d172d81fa3c344e99cf635265ea78472 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/d172d81fa3c344e99cf635265ea78472 2024-11-18T03:48:16,805 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/d172d81fa3c344e99cf635265ea78472, delay=942 ms 2024-11-18T03:48:16,807 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/d172d81fa3c344e99cf635265ea78472, entries=2, sequenceid=15, filesize=5.0 K 2024-11-18T03:48:16,807 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/table/921868a6c61549009ac0350e4dc80e3f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/921868a6c61549009ac0350e4dc80e3f 2024-11-18T03:48:16,808 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/921868a6c61549009ac0350e4dc80e3f, delay=906 ms 2024-11-18T03:48:16,809 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/921868a6c61549009ac0350e4dc80e3f, entries=2, sequenceid=15, filesize=5.3 K 2024-11-18T03:48:16,810 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=15, compaction requested=false 2024-11-18T03:48:16,810 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-18T03:48:16,812 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/b6a4699159f44dd99b89eeb5d6148af8 2024-11-18T03:48:16,812 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/d172d81fa3c344e99cf635265ea78472 2024-11-18T03:48:16,813 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/921868a6c61549009ac0350e4dc80e3f 2024-11-18T03:48:16,819 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-18T03:48:16,819 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T03:48:16,819 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:16,820 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731901696706Running coprocessor pre-close hooks at 1731901696707 (+1 ms)Disabling compacts and flushes for region at 1731901696707Disabling writes for close at 1731901696707Obtaining lock to block concurrent updates at 1731901696707Preparing flush snapshotting stores in 1588230740 at 1731901696707Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731901696708 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731901696708Flushing 1588230740/info: creating writer at 1731901696709 (+1 ms)Flushing 1588230740/info: appending metadata at 1731901696736 (+27 ms)Flushing 1588230740/info: closing flushed file at 1731901696736Flushing 1588230740/ns: creating writer at 1731901696743 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731901696772 (+29 ms)Flushing 1588230740/ns: closing flushed file at 1731901696772Flushing 1588230740/table: creating writer at 1731901696778 (+6 ms)Flushing 1588230740/table: appending metadata at 1731901696798 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731901696798Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e3550f1: reopening flushed file at 1731901696801 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77036c4e: reopening flushed file at 1731901696804 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14ba62e5: reopening flushed file at 1731901696807 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=15, compaction requested=false at 1731901696810 (+3 ms)Writing region close event to WAL at 1731901696813 (+3 ms)Running coprocessor post-close hooks at 1731901696819 (+6 ms)Closed at 1731901696819 2024-11-18T03:48:16,820 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:16,907 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(976): stopping server 88c1dcece5ca,35609,1731901692024; all regions closed. 2024-11-18T03:48:16,907 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(976): stopping server 88c1dcece5ca,36815,1731901692167; all regions closed. 2024-11-18T03:48:16,912 DEBUG [RS:0;88c1dcece5ca:35609 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs 2024-11-18T03:48:16,912 DEBUG [RS:1;88c1dcece5ca:36815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C35609%2C1731901692024:(num 1731901693554) 2024-11-18T03:48:16,913 INFO [RS:1;88c1dcece5ca:36815 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C36815%2C1731901692167.meta:.meta(num 1731901693973) 2024-11-18T03:48:16,913 DEBUG [RS:0;88c1dcece5ca:35609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.ChoreService(370): Chore service for: regionserver/88c1dcece5ca:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T03:48:16,913 INFO [regionserver/88c1dcece5ca:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T03:48:16,913 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:16,914 INFO [RS:0;88c1dcece5ca:35609 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35609 2024-11-18T03:48:16,915 DEBUG [RS:1;88c1dcece5ca:36815 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs 2024-11-18T03:48:16,915 INFO [RS:1;88c1dcece5ca:36815 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C36815%2C1731901692167:(num 1731901693554) 2024-11-18T03:48:16,915 DEBUG [RS:1;88c1dcece5ca:36815 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:16,915 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:16,915 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:16,915 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.ChoreService(370): Chore service for: regionserver/88c1dcece5ca:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:16,915 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:16,915 INFO [regionserver/88c1dcece5ca:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:16,916 INFO [RS:1;88c1dcece5ca:36815 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36815 2024-11-18T03:48:16,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88c1dcece5ca,36815,1731901692167 2024-11-18T03:48:16,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88c1dcece5ca,35609,1731901692024 2024-11-18T03:48:16,918 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:16,918 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:16,918 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:16,921 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88c1dcece5ca,35609,1731901692024] 2024-11-18T03:48:16,923 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88c1dcece5ca,35609,1731901692024 already deleted, retry=false 2024-11-18T03:48:16,923 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88c1dcece5ca,35609,1731901692024 expired; onlineServers=1 2024-11-18T03:48:16,923 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88c1dcece5ca,36815,1731901692167] 2024-11-18T03:48:16,925 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88c1dcece5ca,36815,1731901692167 already deleted, retry=false 2024-11-18T03:48:16,925 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88c1dcece5ca,36815,1731901692167 expired; onlineServers=0 2024-11-18T03:48:16,925 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '88c1dcece5ca,34237,1731901691239' ***** 2024-11-18T03:48:16,925 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T03:48:16,925 INFO [M:0;88c1dcece5ca:34237 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:16,925 INFO [M:0;88c1dcece5ca:34237 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:16,925 DEBUG [M:0;88c1dcece5ca:34237 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T03:48:16,925 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T03:48:16,926 DEBUG [M:0;88c1dcece5ca:34237 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T03:48:16,926 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901693218 {}] cleaner.HFileCleaner(306): Exit Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901693218,5,FailOnTimeoutGroup] 2024-11-18T03:48:16,926 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901693216 {}] cleaner.HFileCleaner(306): Exit Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901693216,5,FailOnTimeoutGroup] 2024-11-18T03:48:16,926 INFO [M:0;88c1dcece5ca:34237 {}] hbase.ChoreService(370): Chore service for: master/88c1dcece5ca:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:16,926 INFO [M:0;88c1dcece5ca:34237 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:16,926 DEBUG [M:0;88c1dcece5ca:34237 {}] master.HMaster(1795): Stopping service threads 2024-11-18T03:48:16,926 INFO [M:0;88c1dcece5ca:34237 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T03:48:16,926 INFO [M:0;88c1dcece5ca:34237 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T03:48:16,927 INFO [M:0;88c1dcece5ca:34237 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T03:48:16,927 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T03:48:16,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:16,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:16,928 DEBUG [M:0;88c1dcece5ca:34237 {}] zookeeper.ZKUtil(347): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T03:48:16,928 WARN [M:0;88c1dcece5ca:34237 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T03:48:16,928 INFO [M:0;88c1dcece5ca:34237 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.lastflushedseqids 2024-11-18T03:48:16,938 INFO [M:0;88c1dcece5ca:34237 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T03:48:16,938 INFO [M:0;88c1dcece5ca:34237 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T03:48:16,938 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:48:16,938 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:16,939 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:16,939 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:48:16,939 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:16,939 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-18T03:48:16,949 DEBUG [M:0;88c1dcece5ca:34237 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e967301d52604ced8336be5549d401d8 2024-11-18T03:48:16,963 DEBUG [M:0;88c1dcece5ca:34237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e967301d52604ced8336be5549d401d8 is 82, key is hbase:meta,,1/info:regioninfo/1731901694037/Put/seqid=0 2024-11-18T03:48:16,963 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e967301d52604ced8336be5549d401d8 2024-11-18T03:48:16,972 DEBUG [M:0;88c1dcece5ca:34237 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ac965aafb99245d18a013f560420d4c4 2024-11-18T03:48:16,986 DEBUG [M:0;88c1dcece5ca:34237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac965aafb99245d18a013f560420d4c4 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731901694725/Put/seqid=0 2024-11-18T03:48:16,987 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac965aafb99245d18a013f560420d4c4 2024-11-18T03:48:16,995 DEBUG [M:0;88c1dcece5ca:34237 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4ece08df82cf4aada04b36756e37cadf 2024-11-18T03:48:17,009 DEBUG [M:0;88c1dcece5ca:34237 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ece08df82cf4aada04b36756e37cadf is 69, key is 88c1dcece5ca,35609,1731901692024/rs:state/1731901693287/Put/seqid=0 2024-11-18T03:48:17,010 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ece08df82cf4aada04b36756e37cadf 2024-11-18T03:48:17,011 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e967301d52604ced8336be5549d401d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e967301d52604ced8336be5549d401d8 2024-11-18T03:48:17,013 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e967301d52604ced8336be5549d401d8, entries=8, sequenceid=84, filesize=5.5 K 2024-11-18T03:48:17,014 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ac965aafb99245d18a013f560420d4c4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ac965aafb99245d18a013f560420d4c4 2024-11-18T03:48:17,015 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ac965aafb99245d18a013f560420d4c4, entries=9, sequenceid=84, filesize=6.5 K 2024-11-18T03:48:17,015 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/4ece08df82cf4aada04b36756e37cadf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4ece08df82cf4aada04b36756e37cadf 2024-11-18T03:48:17,017 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/4ece08df82cf4aada04b36756e37cadf, entries=2, sequenceid=84, filesize=5.1 K 2024-11-18T03:48:17,018 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=84, compaction requested=false 2024-11-18T03:48:17,020 INFO [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:17,020 DEBUG [M:0;88c1dcece5ca:34237 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731901696938Disabling compacts and flushes for region at 1731901696938Disabling writes for close at 1731901696939 (+1 ms)Obtaining lock to block concurrent updates at 1731901696939Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731901696939Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731901696939Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731901696940 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731901696940Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731901696962 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731901696962Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731901696966 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731901696986 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731901696986Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731901696989 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731901697008 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731901697008Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7171b6d8: reopening flushed file at 1731901697011 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e23ce5e: reopening flushed file at 1731901697013 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e6ab957: reopening flushed file at 1731901697015 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=84, compaction requested=false at 1731901697018 (+3 ms)Writing region close event to WAL at 1731901697020 (+2 ms)Closed at 1731901697020 2024-11-18T03:48:17,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:17,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36815-0x10129307eb40002, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:17,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:17,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35609-0x10129307eb40001, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:17,021 INFO [RS:1;88c1dcece5ca:36815 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:17,022 INFO [RS:1;88c1dcece5ca:36815 {}] regionserver.HRegionServer(1031): Exiting; stopping=88c1dcece5ca,36815,1731901692167; zookeeper connection closed. 2024-11-18T03:48:17,022 INFO [RS:0;88c1dcece5ca:35609 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:17,022 INFO [RS:0;88c1dcece5ca:35609 {}] regionserver.HRegionServer(1031): Exiting; stopping=88c1dcece5ca,35609,1731901692024; zookeeper connection closed. 2024-11-18T03:48:17,022 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22e4b1bc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22e4b1bc 2024-11-18T03:48:17,022 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:17,022 INFO [M:0;88c1dcece5ca:34237 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T03:48:17,023 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@eb1260a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@eb1260a 2024-11-18T03:48:17,023 INFO [M:0;88c1dcece5ca:34237 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34237 2024-11-18T03:48:17,023 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-18T03:48:17,023 INFO [M:0;88c1dcece5ca:34237 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:17,126 INFO [M:0;88c1dcece5ca:34237 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:17,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:17,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34237-0x10129307eb40000, quorum=127.0.0.1:62657, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:17,131 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T03:48:17,152 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T03:48:17,153 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,160 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/88c1dcece5ca:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731901696630 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731901696805 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731901695100 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731901696802 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/88c1dcece5ca:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/88c1dcece5ca:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/88c1dcece5ca:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=627 (was 665), ProcessCount=11 (was 11), AvailableMemoryMB=5490 (was 5849) 2024-11-18T03:48:17,166 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=627, ProcessCount=11, AvailableMemoryMB=5490 2024-11-18T03:48:17,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/hadoop-log-dir so I do NOT create it in target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c 2024-11-18T03:48:17,167 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/hadoop-tmp-dir so I do NOT create it in target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c 2024-11-18T03:48:17,167 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,170 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/cluster_4cccc997-8000-f898-7cc9-33e554869fe8, deleteOnExit=true 2024-11-18T03:48:17,176 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/cluster_4cccc997-8000-f898-7cc9-33e554869fe8/zookeeper_0, clientPort=61427, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/cluster_4cccc997-8000-f898-7cc9-33e554869fe8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/cluster_4cccc997-8000-f898-7cc9-33e554869fe8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-18T03:48:17,177 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61427 2024-11-18T03:48:17,188 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a with version=8 2024-11-18T03:48:17,188 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/hbase-staging 2024-11-18T03:48:17,191 INFO [Time-limited test {}] client.ConnectionUtils(128): master/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:17,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,191 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:17,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,191 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:17,191 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-18T03:48:17,192 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:17,192 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43223 2024-11-18T03:48:17,194 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43223 connecting to ZooKeeper ensemble=127.0.0.1:61427 2024-11-18T03:48:17,200 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432230x0, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:17,201 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43223-0x101293097240000 connected 2024-11-18T03:48:17,217 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,217 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:17,219 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a, hbase.cluster.distributed=false 2024-11-18T03:48:17,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:17,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43223 2024-11-18T03:48:17,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43223 2024-11-18T03:48:17,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43223 2024-11-18T03:48:17,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43223 2024-11-18T03:48:17,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43223 2024-11-18T03:48:17,239 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:17,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,239 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:17,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,239 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:17,239 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T03:48:17,239 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:17,240 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37391 2024-11-18T03:48:17,241 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37391 connecting to ZooKeeper ensemble=127.0.0.1:61427 2024-11-18T03:48:17,241 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,242 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373910x0, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:17,245 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37391-0x101293097240001 connected 2024-11-18T03:48:17,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:17,245 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T03:48:17,246 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.cache 2024-11-18T03:48:17,246 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T03:48:17,246 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T03:48:17,246 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T03:48:17,246 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T03:48:17,247 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T03:48:17,247 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T03:48:17,247 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T03:48:17,247 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T03:48:17,247 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence is complete. 2024-11-18T03:48:17,248 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T03:48:17,249 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:17,249 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37391 2024-11-18T03:48:17,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37391 2024-11-18T03:48:17,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37391 2024-11-18T03:48:17,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37391 2024-11-18T03:48:17,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37391 2024-11-18T03:48:17,268 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:17,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,268 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:17,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:17,269 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:17,269 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T03:48:17,269 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:17,270 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33811 2024-11-18T03:48:17,271 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33811 connecting to ZooKeeper ensemble=127.0.0.1:61427 2024-11-18T03:48:17,271 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,272 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,275 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:338110x0, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:17,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:338110x0, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:17,275 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33811-0x101293097240002 connected 2024-11-18T03:48:17,276 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T03:48:17,276 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.cache 2024-11-18T03:48:17,277 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T03:48:17,277 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T03:48:17,277 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T03:48:17,280 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T03:48:17,280 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T03:48:17,281 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-18T03:48:17,281 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T03:48:17,281 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence is complete. 2024-11-18T03:48:17,284 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T03:48:17,285 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-18T03:48:17,286 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:17,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33811 2024-11-18T03:48:17,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33811 2024-11-18T03:48:17,287 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33811 2024-11-18T03:48:17,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33811 2024-11-18T03:48:17,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33811 2024-11-18T03:48:17,301 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;88c1dcece5ca:43223 2024-11-18T03:48:17,301 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:17,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:17,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:17,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:17,305 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:17,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:17,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:17,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,308 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T03:48:17,309 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/88c1dcece5ca,43223,1731901697190 from backup master directory 2024-11-18T03:48:17,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:17,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:17,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:17,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-18T03:48:17,311 WARN [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:17,311 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:17,312 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/hbase.id] with ID: b4f086f7-ab36-4e87-8f22-771b0f15d573 2024-11-18T03:48:17,312 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.tmp/hbase.id 2024-11-18T03:48:17,316 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/hbase.id] 2024-11-18T03:48:17,351 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:17,351 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-18T03:48:17,353 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-18T03:48:17,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,360 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:48:17,361 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-18T03:48:17,362 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:17,382 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store 2024-11-18T03:48:17,389 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:17,389 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:48:17,389 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:17,390 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:17,390 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:48:17,390 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:17,390 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:17,390 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731901697389Disabling compacts and flushes for region at 1731901697389Disabling writes for close at 1731901697390 (+1 ms)Writing region close event to WAL at 1731901697390Closed at 1731901697390 2024-11-18T03:48:17,390 WARN [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/.initializing 2024-11-18T03:48:17,390 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/WALs/88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:17,396 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:17,400 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=88c1dcece5ca%2C43223%2C1731901697190, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/WALs/88c1dcece5ca,43223,1731901697190, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/oldWALs, maxLogs=10 2024-11-18T03:48:17,417 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/WALs/88c1dcece5ca,43223,1731901697190/88c1dcece5ca%2C43223%2C1731901697190.1731901697400 2024-11-18T03:48:17,418 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:17,418 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:17,418 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:17,418 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,418 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,419 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-18T03:48:17,424 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:17,424 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-18T03:48:17,428 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:17,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-18T03:48:17,432 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:17,432 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-18T03:48:17,436 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,436 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:17,436 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,437 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,437 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,437 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,437 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,438 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:17,438 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-18T03:48:17,445 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:17,445 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74782892, jitterRate=0.11435192823410034}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:17,447 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731901697419Initializing all the Stores at 1731901697419Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901697419Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901697419Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901697419Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901697419Cleaning up temporary data from old regions at 1731901697437 (+18 ms)Region opened successfully at 1731901697446 (+9 ms) 2024-11-18T03:48:17,447 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-18T03:48:17,451 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@548c3c0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:17,453 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-18T03:48:17,453 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-18T03:48:17,453 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-18T03:48:17,453 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-18T03:48:17,454 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-18T03:48:17,454 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-18T03:48:17,454 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-18T03:48:17,456 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-18T03:48:17,457 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-18T03:48:17,460 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-18T03:48:17,460 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-18T03:48:17,461 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-18T03:48:17,462 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-18T03:48:17,463 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-18T03:48:17,464 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-18T03:48:17,465 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-18T03:48:17,466 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-18T03:48:17,467 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-18T03:48:17,470 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-18T03:48:17,472 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-18T03:48:17,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:17,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:17,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:17,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,474 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,475 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=88c1dcece5ca,43223,1731901697190, sessionid=0x101293097240000, setting cluster-up flag (Was=false) 2024-11-18T03:48:17,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,478 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,484 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-18T03:48:17,485 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:17,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,493 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-18T03:48:17,494 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:17,495 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-18T03:48:17,496 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:17,497 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-18T03:48:17,497 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-18T03:48:17,497 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 88c1dcece5ca,43223,1731901697190 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=5, maxPoolSize=5 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/88c1dcece5ca:0, corePoolSize=10, maxPoolSize=10 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:17,499 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,500 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731901727500 2024-11-18T03:48:17,500 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-18T03:48:17,501 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,501 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-18T03:48:17,501 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,501 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-18T03:48:17,502 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-18T03:48:17,502 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-18T03:48:17,503 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-18T03:48:17,503 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-18T03:48:17,503 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901697503,5,FailOnTimeoutGroup] 2024-11-18T03:48:17,503 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901697503,5,FailOnTimeoutGroup] 2024-11-18T03:48:17,503 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,504 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-18T03:48:17,504 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,504 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,520 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-18T03:48:17,520 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:17,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:17,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T03:48:17,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:48:17,532 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:17,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T03:48:17,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:48:17,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:17,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T03:48:17,539 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:48:17,539 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:17,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T03:48:17,543 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:48:17,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:17,543 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:17,543 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:48:17,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:17,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:17,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:48:17,544 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:48:17,544 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:17,545 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:48:17,550 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:17,550 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73976296, jitterRate=0.1023327112197876}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:17,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731901697528Initializing all the Stores at 1731901697528Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901697528Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901697528Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901697528Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901697528Cleaning up temporary data from old regions at 1731901697544 (+16 ms)Region opened successfully at 1731901697551 (+7 ms) 2024-11-18T03:48:17,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:48:17,551 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:48:17,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:48:17,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:48:17,551 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:48:17,552 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:17,552 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731901697551Disabling compacts and flushes for region at 1731901697551Disabling writes for close at 1731901697551Writing region close event to WAL at 1731901697552 (+1 ms)Closed at 1731901697552 2024-11-18T03:48:17,553 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:17,553 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-18T03:48:17,553 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-18T03:48:17,554 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:48:17,555 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-18T03:48:17,590 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(746): ClusterId : b4f086f7-ab36-4e87-8f22-771b0f15d573 2024-11-18T03:48:17,590 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(746): ClusterId : b4f086f7-ab36-4e87-8f22-771b0f15d573 2024-11-18T03:48:17,590 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T03:48:17,591 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T03:48:17,594 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T03:48:17,594 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T03:48:17,594 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T03:48:17,594 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T03:48:17,597 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T03:48:17,597 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T03:48:17,597 DEBUG [RS:1;88c1dcece5ca:33811 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ed84b26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:17,598 DEBUG [RS:0;88c1dcece5ca:37391 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47309172, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:17,610 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;88c1dcece5ca:33811 2024-11-18T03:48:17,610 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T03:48:17,610 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T03:48:17,610 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T03:48:17,611 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(2659): reportForDuty to master=88c1dcece5ca,43223,1731901697190 with port=33811, startcode=1731901697268 2024-11-18T03:48:17,611 DEBUG [RS:1;88c1dcece5ca:33811 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T03:48:17,614 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43737, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T03:48:17,614 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43223 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:17,614 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43223 {}] master.ServerManager(517): Registering regionserver=88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:17,615 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;88c1dcece5ca:37391 2024-11-18T03:48:17,615 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T03:48:17,615 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T03:48:17,615 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T03:48:17,616 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:17,616 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T03:48:17,616 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T03:48:17,616 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(2659): reportForDuty to master=88c1dcece5ca,43223,1731901697190 with port=37391, startcode=1731901697238 2024-11-18T03:48:17,616 DEBUG [RS:0;88c1dcece5ca:37391 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T03:48:17,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:17,618 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54319, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T03:48:17,619 DEBUG [RS:1;88c1dcece5ca:33811 {}] zookeeper.ZKUtil(111): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:17,619 WARN [RS:1;88c1dcece5ca:33811 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:17,619 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43223 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:17,619 INFO [RS:1;88c1dcece5ca:33811 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:17,619 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43223 {}] master.ServerManager(517): Registering regionserver=88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:17,619 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:17,619 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88c1dcece5ca,33811,1731901697268] 2024-11-18T03:48:17,621 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:17,621 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T03:48:17,621 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T03:48:17,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:17,628 DEBUG [RS:0;88c1dcece5ca:37391 {}] zookeeper.ZKUtil(111): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:17,628 WARN [RS:0;88c1dcece5ca:37391 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:17,628 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88c1dcece5ca,37391,1731901697238] 2024-11-18T03:48:17,628 INFO [RS:0;88c1dcece5ca:37391 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:17,628 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:17,629 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T03:48:17,632 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T03:48:17,633 INFO [RS:1;88c1dcece5ca:33811 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T03:48:17,633 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,633 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T03:48:17,633 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T03:48:17,634 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T03:48:17,634 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,634 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,634 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,635 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,636 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:17,636 DEBUG [RS:1;88c1dcece5ca:33811 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:17,636 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T03:48:17,637 INFO [RS:0;88c1dcece5ca:37391 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T03:48:17,637 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,637 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,637 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T03:48:17,637 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,638 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,638 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,638 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,638 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,33811,1731901697268-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:17,638 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T03:48:17,638 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,638 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,638 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,638 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,638 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:17,639 DEBUG [RS:0;88c1dcece5ca:37391 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:17,640 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,641 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,641 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,641 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,641 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,641 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,37391,1731901697238-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:17,657 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T03:48:17,657 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,37391,1731901697238-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,658 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,658 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.Replication(171): 88c1dcece5ca,37391,1731901697238 started 2024-11-18T03:48:17,659 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T03:48:17,659 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,33811,1731901697268-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,660 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,660 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.Replication(171): 88c1dcece5ca,33811,1731901697268 started 2024-11-18T03:48:17,672 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,673 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1482): Serving as 88c1dcece5ca,37391,1731901697238, RpcServer on 88c1dcece5ca/172.17.0.2:37391, sessionid=0x101293097240001 2024-11-18T03:48:17,673 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T03:48:17,673 DEBUG [RS:0;88c1dcece5ca:37391 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:17,673 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,37391,1731901697238' 2024-11-18T03:48:17,673 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T03:48:17,674 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T03:48:17,674 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T03:48:17,674 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T03:48:17,674 DEBUG [RS:0;88c1dcece5ca:37391 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:17,674 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,37391,1731901697238' 2024-11-18T03:48:17,674 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T03:48:17,675 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T03:48:17,675 DEBUG [RS:0;88c1dcece5ca:37391 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T03:48:17,675 INFO [RS:0;88c1dcece5ca:37391 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T03:48:17,675 INFO [RS:0;88c1dcece5ca:37391 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T03:48:17,679 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:17,680 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1482): Serving as 88c1dcece5ca,33811,1731901697268, RpcServer on 88c1dcece5ca/172.17.0.2:33811, sessionid=0x101293097240002 2024-11-18T03:48:17,680 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T03:48:17,680 DEBUG [RS:1;88c1dcece5ca:33811 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:17,680 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,33811,1731901697268' 2024-11-18T03:48:17,680 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T03:48:17,680 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T03:48:17,681 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T03:48:17,681 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T03:48:17,681 DEBUG [RS:1;88c1dcece5ca:33811 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:17,681 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,33811,1731901697268' 2024-11-18T03:48:17,681 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T03:48:17,681 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T03:48:17,682 DEBUG [RS:1;88c1dcece5ca:33811 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T03:48:17,682 INFO [RS:1;88c1dcece5ca:33811 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T03:48:17,682 INFO [RS:1;88c1dcece5ca:33811 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T03:48:17,705 WARN [88c1dcece5ca:43223 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-18T03:48:17,776 INFO [RS:0;88c1dcece5ca:37391 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:17,777 INFO [RS:0;88c1dcece5ca:37391 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C37391%2C1731901697238, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37391,1731901697238, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs, maxLogs=55 2024-11-18T03:48:17,782 INFO [RS:1;88c1dcece5ca:33811 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:17,783 INFO [RS:1;88c1dcece5ca:33811 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C33811%2C1731901697268, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,33811,1731901697268, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs, maxLogs=55 2024-11-18T03:48:17,796 INFO [RS:0;88c1dcece5ca:37391 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37391,1731901697238/88c1dcece5ca%2C37391%2C1731901697238.1731901697778 2024-11-18T03:48:17,800 DEBUG [RS:0;88c1dcece5ca:37391 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:17,804 INFO [RS:1;88c1dcece5ca:33811 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,33811,1731901697268/88c1dcece5ca%2C33811%2C1731901697268.1731901697784 2024-11-18T03:48:17,805 DEBUG [RS:1;88c1dcece5ca:33811 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:17,955 DEBUG [88c1dcece5ca:43223 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-18T03:48:17,956 DEBUG [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(204): Hosts are {88c1dcece5ca=0} racks are {/default-rack=0} 2024-11-18T03:48:17,956 DEBUG [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T03:48:17,956 DEBUG [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T03:48:17,956 DEBUG [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T03:48:17,956 DEBUG [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T03:48:17,956 INFO [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T03:48:17,956 INFO [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T03:48:17,956 DEBUG [88c1dcece5ca:43223 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T03:48:17,957 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:17,958 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88c1dcece5ca,33811,1731901697268, state=OPENING 2024-11-18T03:48:17,959 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-18T03:48:17,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:17,962 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-18T03:48:17,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:17,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:17,962 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:17,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=88c1dcece5ca,33811,1731901697268}] 2024-11-18T03:48:18,114 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T03:48:18,117 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37533, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T03:48:18,119 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-18T03:48:18,119 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:18,120 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-18T03:48:18,121 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C33811%2C1731901697268.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,33811,1731901697268, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs, maxLogs=55 2024-11-18T03:48:18,142 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,33811,1731901697268/88c1dcece5ca%2C33811%2C1731901697268.meta.1731901698122.meta 2024-11-18T03:48:18,142 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:18,142 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:18,143 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-18T03:48:18,143 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-18T03:48:18,143 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-18T03:48:18,143 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-18T03:48:18,143 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:18,143 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-18T03:48:18,143 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-18T03:48:18,144 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-18T03:48:18,145 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-18T03:48:18,145 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:18,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:18,146 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-18T03:48:18,146 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-18T03:48:18,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:18,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:18,147 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-18T03:48:18,147 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-18T03:48:18,147 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:18,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:18,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-18T03:48:18,148 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-18T03:48:18,148 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:18,148 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-18T03:48:18,149 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-18T03:48:18,149 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:18,150 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740 2024-11-18T03:48:18,150 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-18T03:48:18,150 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-18T03:48:18,150 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-18T03:48:18,151 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-18T03:48:18,152 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62499570, jitterRate=-0.06868383288383484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-18T03:48:18,152 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-18T03:48:18,153 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731901698144Writing region info on filesystem at 1731901698144Initializing all the Stores at 1731901698144Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901698144Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901698144Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901698144Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731901698144Cleaning up temporary data from old regions at 1731901698150 (+6 ms)Running coprocessor post-open hooks at 1731901698152 (+2 ms)Region opened successfully at 1731901698153 (+1 ms) 2024-11-18T03:48:18,153 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731901698114 2024-11-18T03:48:18,156 DEBUG [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-18T03:48:18,156 INFO [RS_OPEN_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-18T03:48:18,156 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:18,157 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 88c1dcece5ca,33811,1731901697268, state=OPEN 2024-11-18T03:48:18,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:18,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:18,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-18T03:48:18,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:18,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:18,159 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:18,159 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-18T03:48:18,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-18T03:48:18,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=88c1dcece5ca,33811,1731901697268 in 197 msec 2024-11-18T03:48:18,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-18T03:48:18,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-18T03:48:18,164 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-18T03:48:18,164 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-18T03:48:18,168 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:48:18,168 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88c1dcece5ca,33811,1731901697268, seqNum=-1] 2024-11-18T03:48:18,168 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:48:18,170 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48817, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:48:18,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 679 msec 2024-11-18T03:48:18,175 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731901698175, completionTime=-1 2024-11-18T03:48:18,175 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-18T03:48:18,175 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731901758178 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731901818178 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,43223,1731901697190-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,43223,1731901697190-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,43223,1731901697190-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-88c1dcece5ca:43223, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:18,178 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:18,179 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:18,181 DEBUG [master/88c1dcece5ca:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-18T03:48:18,183 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.872sec 2024-11-18T03:48:18,184 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-18T03:48:18,184 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-18T03:48:18,184 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-18T03:48:18,184 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-18T03:48:18,184 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-18T03:48:18,184 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,43223,1731901697190-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:18,184 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,43223,1731901697190-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-18T03:48:18,185 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-18T03:48:18,185 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-18T03:48:18,185 INFO [master/88c1dcece5ca:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,43223,1731901697190-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:18,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:18,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 88c1dcece5ca,43223,-1 for getting cluster id 2024-11-18T03:48:18,190 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-18T03:48:18,192 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b4f086f7-ab36-4e87-8f22-771b0f15d573' 2024-11-18T03:48:18,192 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-18T03:48:18,192 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b4f086f7-ab36-4e87-8f22-771b0f15d573" 2024-11-18T03:48:18,193 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f7b3933, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:18,193 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [88c1dcece5ca,43223,-1] 2024-11-18T03:48:18,193 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-18T03:48:18,193 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:18,195 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-18T03:48:18,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-18T03:48:18,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-18T03:48:18,197 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=88c1dcece5ca,33811,1731901697268, seqNum=-1] 2024-11-18T03:48:18,198 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:48:18,199 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:48:18,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:18,201 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-18T03:48:18,202 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:18,203 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@55b62dc8 2024-11-18T03:48:18,203 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-18T03:48:18,204 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-18T03:48:18,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43223 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731901698201', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-18T03:48:18,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43223 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 2024-11-18T03:48:18,207 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-18T03:48:18,208 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:18,208 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43223 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731901698201" procId is: 4 2024-11-18T03:48:18,209 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-18T03:48:18,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:18,220 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ff148b60f3b6f5b01616332f09576c8, NAME => 'testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731901698201', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:18,227 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:18,227 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(1722): Closing 1ff148b60f3b6f5b01616332f09576c8, disabling compactions & flushes 2024-11-18T03:48:18,227 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:18,227 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:18,227 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. after waiting 0 ms 2024-11-18T03:48:18,227 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:18,227 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:18,227 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731901698201-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ff148b60f3b6f5b01616332f09576c8: Waiting for close lock at 1731901698227Disabling compacts and flushes for region at 1731901698227Disabling writes for close at 1731901698227Writing region close event to WAL at 1731901698227Closed at 1731901698227 2024-11-18T03:48:18,229 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 execute state=CREATE_TABLE_ADD_TO_META 2024-11-18T03:48:18,229 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731901698229"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731901698229"}]},"ts":"1731901698229"} 2024-11-18T03:48:18,232 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-18T03:48:18,233 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-18T03:48:18,233 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731901698201","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731901698233"}]},"ts":"1731901698233"} 2024-11-18T03:48:18,235 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731901698201, state=ENABLING in hbase:meta 2024-11-18T03:48:18,235 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {88c1dcece5ca=0} racks are {/default-rack=0} 2024-11-18T03:48:18,236 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-18T03:48:18,236 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-18T03:48:18,236 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-18T03:48:18,236 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-18T03:48:18,236 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-18T03:48:18,236 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-18T03:48:18,236 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-18T03:48:18,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN}] 2024-11-18T03:48:18,238 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN 2024-11-18T03:48:18,239 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN; state=OFFLINE, location=88c1dcece5ca,37391,1731901697238; forceNewPlan=false, retain=false 2024-11-18T03:48:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:18,389 INFO [88c1dcece5ca:43223 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-18T03:48:18,390 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1ff148b60f3b6f5b01616332f09576c8, regionState=OPENING, regionLocation=88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:18,393 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN because future has completed 2024-11-18T03:48:18,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ff148b60f3b6f5b01616332f09576c8, server=88c1dcece5ca,37391,1731901697238}] 2024-11-18T03:48:18,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:18,547 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-18T03:48:18,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49551, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-18T03:48:18,551 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:18,551 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ff148b60f3b6f5b01616332f09576c8, NAME => 'testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:18,552 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731901698201 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,552 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:18,552 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,552 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,553 INFO [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,557 INFO [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ff148b60f3b6f5b01616332f09576c8 columnFamilyName family 2024-11-18T03:48:18,557 DEBUG [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:18,557 INFO [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] regionserver.HStore(327): Store=1ff148b60f3b6f5b01616332f09576c8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:18,557 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,558 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,558 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,558 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,558 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,559 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,566 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-18T03:48:18,566 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1ff148b60f3b6f5b01616332f09576c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61777943, jitterRate=-0.07943691313266754}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T03:48:18,566 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:18,567 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1ff148b60f3b6f5b01616332f09576c8: Running coprocessor pre-open hook at 1731901698552Writing region info on filesystem at 1731901698552Initializing all the Stores at 1731901698553 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901698553Cleaning up temporary data from old regions at 1731901698558 (+5 ms)Running coprocessor post-open hooks at 1731901698566 (+8 ms)Region opened successfully at 1731901698567 (+1 ms) 2024-11-18T03:48:18,568 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8., pid=6, masterSystemTime=1731901698547 2024-11-18T03:48:18,571 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:18,571 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:18,571 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1ff148b60f3b6f5b01616332f09576c8, regionState=OPEN, openSeqNum=2, regionLocation=88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:18,574 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ff148b60f3b6f5b01616332f09576c8, server=88c1dcece5ca,37391,1731901697238 because future has completed 2024-11-18T03:48:18,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-18T03:48:18,577 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ff148b60f3b6f5b01616332f09576c8, server=88c1dcece5ca,37391,1731901697238 in 181 msec 2024-11-18T03:48:18,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-18T03:48:18,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN in 340 msec 2024-11-18T03:48:18,580 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-18T03:48:18,580 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731901698201","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731901698580"}]},"ts":"1731901698580"} 2024-11-18T03:48:18,582 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731901698201, state=ENABLED in hbase:meta 2024-11-18T03:48:18,583 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 execute state=CREATE_TABLE_POST_OPERATION 2024-11-18T03:48:18,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731901698201 in 378 msec 2024-11-18T03:48:18,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43223 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-18T03:48:18,833 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731901698201 completed 2024-11-18T03:48:18,833 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731901698201 get assigned. Timeout = 60000ms 2024-11-18T03:48:18,833 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T03:48:18,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731901698201 assigned to meta. Checking AM states. 2024-11-18T03:48:18,836 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-18T03:48:18,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731901698201 assigned. 2024-11-18T03:48:18,840 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731901698201', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8., hostname=88c1dcece5ca,37391,1731901697238, seqNum=2] 2024-11-18T03:48:18,840 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-18T03:48:18,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-18T03:48:18,847 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1ff148b60f3b6f5b01616332f09576c8 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-18T03:48:18,860 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 192f69d2a8074f0cbd43aca485a9e4e0 2024-11-18T03:48:18,873 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/.tmp/family/192f69d2a8074f0cbd43aca485a9e4e0 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-18T03:48:18,874 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/.tmp/family/192f69d2a8074f0cbd43aca485a9e4e0 2024-11-18T03:48:18,877 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/.tmp/family/192f69d2a8074f0cbd43aca485a9e4e0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0 2024-11-18T03:48:18,878 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0, delay=938 ms 2024-11-18T03:48:18,879 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0, entries=2, sequenceid=6, filesize=4.9 K 2024-11-18T03:48:18,880 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1ff148b60f3b6f5b01616332f09576c8 in 33ms, sequenceid=6, compaction requested=false 2024-11-18T03:48:18,880 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731901698201' 2024-11-18T03:48:18,881 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1ff148b60f3b6f5b01616332f09576c8: 2024-11-18T03:48:19,507 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-18T03:48:19,817 DEBUG [hfile-prefetch-1731901696630 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 192f69d2a8074f0cbd43aca485a9e4e0_0. 2024-11-18T03:48:19,818 DEBUG [hfile-prefetch-1731901696630 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 192f69d2a8074f0cbd43aca485a9e4e0_117. 2024-11-18T03:48:19,818 DEBUG [hfile-prefetch-1731901696630 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0, with total blocks 2, and data blocks 1 2024-11-18T03:48:19,818 DEBUG [hfile-prefetch-1731901696630 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-18T03:48:19,818 DEBUG [hfile-prefetch-1731901696630 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-18T03:48:19,818 DEBUG [hfile-prefetch-1731901696630 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0. Will sleep 100ms and try the verification again. 2024-11-18T03:48:19,848 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-18T03:48:19,848 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-18T03:48:19,848 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:19,848 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:19,852 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T03:48:19,852 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731901699848 2024-11-18T03:48:19,852 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-18T03:48:19,881 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;88c1dcece5ca:37391,5,FailOnTimeoutGroup] 2024-11-18T03:48:19,881 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88c1dcece5ca,37391,1731901697238' ***** 2024-11-18T03:48:19,881 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-18T03:48:19,882 INFO [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T03:48:19,882 INFO [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence 2024-11-18T03:48:19,882 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T03:48:19,882 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T03:48:19,882 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T03:48:19,882 INFO [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T03:48:19,882 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T03:48:19,892 DEBUG [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1487): Thread RS:0;88c1dcece5ca:37391 started persisting bucket cache to file 2024-11-18T03:48:19,892 DEBUG [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:19,892 DEBUG [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:19,896 DEBUG [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T03:48:19,896 DEBUG [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731901699892 2024-11-18T03:48:19,896 DEBUG [RS:0;88c1dcece5ca:37391 {}] bucket.BucketCache(1509): Thread RS:0;88c1dcece5ca:37391 finished persisting bucket cache to file, renaming 2024-11-18T03:48:19,896 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T03:48:19,896 INFO [RS:0;88c1dcece5ca:37391 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T03:48:19,896 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T03:48:19,896 INFO [RS:0;88c1dcece5ca:37391 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T03:48:19,896 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:19,897 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(959): stopping server 88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:19,897 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:19,897 INFO [RS:0;88c1dcece5ca:37391 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;88c1dcece5ca:37391. 2024-11-18T03:48:19,897 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T03:48:19,897 DEBUG [RS:0;88c1dcece5ca:37391 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:19,897 DEBUG [RS:0;88c1dcece5ca:37391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:19,897 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ff148b60f3b6f5b01616332f09576c8, disabling compactions & flushes 2024-11-18T03:48:19,897 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:19,897 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:19,897 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. after waiting 0 ms 2024-11-18T03:48:19,897 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-18T03:48:19,897 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:19,897 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1325): Online Regions={1ff148b60f3b6f5b01616332f09576c8=testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.} 2024-11-18T03:48:19,897 DEBUG [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1351): Waiting on 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:19,898 DEBUG [StoreFileCloser-1ff148b60f3b6f5b01616332f09576c8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0 2024-11-18T03:48:19,898 WARN [hfile-prefetch-1731901696630 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-18T03:48:19,898 DEBUG [hfile-prefetch-1731901696630 {}] hfile.PrefetchExecutor(121): Prefetch completed for 192f69d2a8074f0cbd43aca485a9e4e0 2024-11-18T03:48:19,902 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-18T03:48:19,902 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:19,903 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ff148b60f3b6f5b01616332f09576c8: Waiting for close lock at 1731901699897Running coprocessor pre-close hooks at 1731901699897Disabling compacts and flushes for region at 1731901699897Disabling writes for close at 1731901699897Writing region close event to WAL at 1731901699898 (+1 ms)Running coprocessor post-close hooks at 1731901699902 (+4 ms)Closed at 1731901699902 2024-11-18T03:48:19,903 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:19,943 INFO [regionserver/88c1dcece5ca:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:20,097 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(976): stopping server 88c1dcece5ca,37391,1731901697238; all regions closed. 2024-11-18T03:48:20,099 DEBUG [RS:0;88c1dcece5ca:37391 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs 2024-11-18T03:48:20,099 INFO [RS:0;88c1dcece5ca:37391 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C37391%2C1731901697238:(num 1731901697778) 2024-11-18T03:48:20,099 DEBUG [RS:0;88c1dcece5ca:37391 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:20,099 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:20,099 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:20,100 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.ChoreService(370): Chore service for: regionserver/88c1dcece5ca:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:20,100 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T03:48:20,100 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T03:48:20,100 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T03:48:20,100 INFO [regionserver/88c1dcece5ca:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:20,100 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:20,100 INFO [RS:0;88c1dcece5ca:37391 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37391 2024-11-18T03:48:20,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88c1dcece5ca,37391,1731901697238 2024-11-18T03:48:20,104 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:20,104 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:20,106 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88c1dcece5ca,37391,1731901697238] 2024-11-18T03:48:20,107 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88c1dcece5ca,37391,1731901697238 already deleted, retry=false 2024-11-18T03:48:20,107 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 88c1dcece5ca,37391,1731901697238 on 88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:20,111 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 88c1dcece5ca,37391,1731901697238, splitWal=true, meta=false 2024-11-18T03:48:20,114 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 88c1dcece5ca,37391,1731901697238, splitWal=true, meta=false 2024-11-18T03:48:20,114 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 88c1dcece5ca,37391,1731901697238 (carryingMeta=false) 88c1dcece5ca,37391,1731901697238/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@717527ab[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-18T03:48:20,115 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 88c1dcece5ca,37391,1731901697238 had 1 regions 2024-11-18T03:48:20,116 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 88c1dcece5ca,37391,1731901697238, splitWal=true, meta=false, isMeta: false 2024-11-18T03:48:20,116 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37391,1731901697238-splitting 2024-11-18T03:48:20,116 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37391,1731901697238-splitting dir is empty, no logs to split. 2024-11-18T03:48:20,117 INFO [PEWorker-2 {}] master.SplitWALManager(105): 88c1dcece5ca,37391,1731901697238 WAL count=0, meta=false 2024-11-18T03:48:20,118 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37391,1731901697238-splitting dir is empty, no logs to split. 2024-11-18T03:48:20,118 INFO [PEWorker-2 {}] master.SplitWALManager(105): 88c1dcece5ca,37391,1731901697238 WAL count=0, meta=false 2024-11-18T03:48:20,118 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 88c1dcece5ca,37391,1731901697238 WAL splitting is done? wals=0, meta=false 2024-11-18T03:48:20,119 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 88c1dcece5ca,37391,1731901697238 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37391,1731901697238-splitting does not exist 2024-11-18T03:48:20,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN}] 2024-11-18T03:48:20,121 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN 2024-11-18T03:48:20,122 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-18T03:48:20,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:20,206 INFO [RS:0;88c1dcece5ca:37391 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:20,206 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37391-0x101293097240001, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:20,206 INFO [RS:0;88c1dcece5ca:37391 {}] regionserver.HRegionServer(1031): Exiting; stopping=88c1dcece5ca,37391,1731901697238; zookeeper connection closed. 2024-11-18T03:48:20,206 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e675fdb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e675fdb 2024-11-18T03:48:20,273 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1ff148b60f3b6f5b01616332f09576c8, regionState=OPENING, regionLocation=88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:20,275 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN because future has completed 2024-11-18T03:48:20,276 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ff148b60f3b6f5b01616332f09576c8, server=88c1dcece5ca,33811,1731901697268}] 2024-11-18T03:48:20,398 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/88c1dcece5ca:0 server-side Connection retries=45 2024-11-18T03:48:20,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:20,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:20,398 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-18T03:48:20,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-18T03:48:20,398 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-18T03:48:20,398 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-18T03:48:20,398 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-18T03:48:20,399 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37717 2024-11-18T03:48:20,400 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37717 connecting to ZooKeeper ensemble=127.0.0.1:61427 2024-11-18T03:48:20,401 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:20,401 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:20,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:377170x0, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-18T03:48:20,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37717-0x101293097240003 connected 2024-11-18T03:48:20,408 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-18T03:48:20,408 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-18T03:48:20,409 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-18T03:48:20,409 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.cache 2024-11-18T03:48:20,409 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-18T03:48:20,409 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-18T03:48:20,409 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-18T03:48:20,410 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-18T03:48:20,410 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-18T03:48:20,410 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-18T03:48:20,411 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-18T03:48:20,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-18T03:48:20,412 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-18T03:48:20,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-18T03:48:20,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37717 2024-11-18T03:48:20,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37717 2024-11-18T03:48:20,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37717 2024-11-18T03:48:20,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37717 2024-11-18T03:48:20,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37717 2024-11-18T03:48:20,415 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(746): ClusterId : b4f086f7-ab36-4e87-8f22-771b0f15d573 2024-11-18T03:48:20,415 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-18T03:48:20,416 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-18T03:48:20,416 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-18T03:48:20,416 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-18T03:48:20,416 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence is complete. 2024-11-18T03:48:20,417 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-18T03:48:20,417 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-18T03:48:20,419 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-18T03:48:20,420 DEBUG [RS:2;88c1dcece5ca:37717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d0cef6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=88c1dcece5ca/172.17.0.2:0 2024-11-18T03:48:20,430 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:20,430 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ff148b60f3b6f5b01616332f09576c8, NAME => 'testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.', STARTKEY => '', ENDKEY => ''} 2024-11-18T03:48:20,431 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731901698201 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,431 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-18T03:48:20,431 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,431 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,431 DEBUG [RS:2;88c1dcece5ca:37717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;88c1dcece5ca:37717 2024-11-18T03:48:20,432 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-18T03:48:20,431 INFO [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,432 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-18T03:48:20,432 DEBUG [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-18T03:48:20,432 INFO [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ff148b60f3b6f5b01616332f09576c8 columnFamilyName family 2024-11-18T03:48:20,432 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(2659): reportForDuty to master=88c1dcece5ca,43223,1731901697190 with port=37717, startcode=1731901700398 2024-11-18T03:48:20,432 DEBUG [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-18T03:48:20,432 DEBUG [RS:2;88c1dcece5ca:37717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-18T03:48:20,434 DEBUG [StoreFileOpener-1ff148b60f3b6f5b01616332f09576c8-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0, delay=915 ms 2024-11-18T03:48:20,434 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49365, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-18T03:48:20,435 DEBUG [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0 2024-11-18T03:48:20,435 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43223 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:20,435 INFO [StoreOpener-1ff148b60f3b6f5b01616332f09576c8-1 {}] regionserver.HStore(327): Store=1ff148b60f3b6f5b01616332f09576c8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-18T03:48:20,435 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43223 {}] master.ServerManager(517): Registering regionserver=88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:20,435 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,436 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,437 DEBUG [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a 2024-11-18T03:48:20,437 DEBUG [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-18T03:48:20,437 DEBUG [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-18T03:48:20,437 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,437 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,437 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:20,438 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,439 DEBUG [RS:2;88c1dcece5ca:37717 {}] zookeeper.ZKUtil(111): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:20,439 WARN [RS:2;88c1dcece5ca:37717 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-18T03:48:20,439 INFO [RS:2;88c1dcece5ca:37717 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-18T03:48:20,439 DEBUG [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:20,439 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [88c1dcece5ca,37717,1731901700398] 2024-11-18T03:48:20,439 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 1ff148b60f3b6f5b01616332f09576c8; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73317320, jitterRate=0.09251320362091064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-18T03:48:20,439 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:20,441 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 1ff148b60f3b6f5b01616332f09576c8: Running coprocessor pre-open hook at 1731901700431Writing region info on filesystem at 1731901700431Initializing all the Stores at 1731901700431Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731901700431Cleaning up temporary data from old regions at 1731901700437 (+6 ms)Running coprocessor post-open hooks at 1731901700439 (+2 ms)Region opened successfully at 1731901700441 (+2 ms) 2024-11-18T03:48:20,442 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8., pid=9, masterSystemTime=1731901700428 2024-11-18T03:48:20,444 DEBUG [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:20,444 INFO [RS_OPEN_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:20,445 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-18T03:48:20,445 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1ff148b60f3b6f5b01616332f09576c8, regionState=OPEN, openSeqNum=10, regionLocation=88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:20,447 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-18T03:48:20,447 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ff148b60f3b6f5b01616332f09576c8, server=88c1dcece5ca,33811,1731901697268 because future has completed 2024-11-18T03:48:20,448 INFO [RS:2;88c1dcece5ca:37717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-18T03:48:20,448 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,448 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-18T03:48:20,449 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-18T03:48:20,449 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,449 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,449 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,449 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,449 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,449 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,449 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/88c1dcece5ca:0, corePoolSize=2, maxPoolSize=2 2024-11-18T03:48:20,449 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,450 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,450 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,450 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,450 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,450 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/88c1dcece5ca:0, corePoolSize=1, maxPoolSize=1 2024-11-18T03:48:20,450 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:20,450 DEBUG [RS:2;88c1dcece5ca:37717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/88c1dcece5ca:0, corePoolSize=3, maxPoolSize=3 2024-11-18T03:48:20,451 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,451 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,451 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-18T03:48:20,451 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,451 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ff148b60f3b6f5b01616332f09576c8, server=88c1dcece5ca,33811,1731901697268 in 173 msec 2024-11-18T03:48:20,451 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,451 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,37717,1731901700398-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-18T03:48:20,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-18T03:48:20,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731901698201, region=1ff148b60f3b6f5b01616332f09576c8, ASSIGN in 331 msec 2024-11-18T03:48:20,453 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 88c1dcece5ca,37391,1731901697238 after splitting done 2024-11-18T03:48:20,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 88c1dcece5ca,37391,1731901697238, splitWal=true, meta=false in 345 msec 2024-11-18T03:48:20,470 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-18T03:48:20,470 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=88c1dcece5ca,37717,1731901700398-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,470 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,470 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.Replication(171): 88c1dcece5ca,37717,1731901700398 started 2024-11-18T03:48:20,485 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-18T03:48:20,485 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(1482): Serving as 88c1dcece5ca,37717,1731901700398, RpcServer on 88c1dcece5ca/172.17.0.2:37717, sessionid=0x101293097240003 2024-11-18T03:48:20,485 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-18T03:48:20,485 DEBUG [RS:2;88c1dcece5ca:37717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:20,485 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,37717,1731901700398' 2024-11-18T03:48:20,485 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-18T03:48:20,486 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-18T03:48:20,486 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-18T03:48:20,486 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-18T03:48:20,486 DEBUG [RS:2;88c1dcece5ca:37717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:20,486 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '88c1dcece5ca,37717,1731901700398' 2024-11-18T03:48:20,486 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-18T03:48:20,487 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-18T03:48:20,487 DEBUG [RS:2;88c1dcece5ca:37717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-18T03:48:20,487 INFO [RS:2;88c1dcece5ca:37717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-18T03:48:20,487 INFO [RS:2;88c1dcece5ca:37717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-18T03:48:20,588 INFO [RS:2;88c1dcece5ca:37717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-18T03:48:20,589 INFO [RS:2;88c1dcece5ca:37717 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=88c1dcece5ca%2C37717%2C1731901700398, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37717,1731901700398, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs, maxLogs=55 2024-11-18T03:48:20,607 INFO [RS:2;88c1dcece5ca:37717 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/WALs/88c1dcece5ca,37717,1731901700398/88c1dcece5ca%2C37717%2C1731901700398.1731901700589 2024-11-18T03:48:20,608 DEBUG [RS:2;88c1dcece5ca:37717 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-18T03:48:20,985 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-18T03:48:20,985 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:48:20,986 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:20,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:20,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:20,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-18T03:48:20,986 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-18T03:48:20,986 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=160297857, stopped=false 2024-11-18T03:48:20,986 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=88c1dcece5ca,43223,1731901697190 2024-11-18T03:48:20,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:20,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:20,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-18T03:48:20,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:20,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:20,990 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:20,990 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T03:48:20,990 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-18T03:48:20,990 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:20,990 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:20,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:20,990 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:20,990 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88c1dcece5ca,33811,1731901697268' ***** 2024-11-18T03:48:20,991 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T03:48:20,991 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '88c1dcece5ca,37717,1731901700398' ***** 2024-11-18T03:48:20,991 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-18T03:48:20,991 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-18T03:48:20,991 INFO [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T03:48:20,991 INFO [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence 2024-11-18T03:48:20,991 INFO [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1738): Disabling cache 2024-11-18T03:48:20,991 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T03:48:20,991 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T03:48:20,991 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T03:48:20,991 INFO [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T03:48:20,991 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-18T03:48:20,991 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-18T03:48:20,991 INFO [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/1fb212f8-8d99-7e1f-2b16-bfa3c800bb2c/bucket.persistence 2024-11-18T03:48:20,992 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-18T03:48:20,992 INFO [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-18T03:48:20,992 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T03:48:20,992 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-18T03:48:21,002 DEBUG [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1487): Thread RS:1;88c1dcece5ca:33811 started persisting bucket cache to file 2024-11-18T03:48:21,002 DEBUG [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:21,002 DEBUG [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:21,002 DEBUG [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1487): Thread RS:2;88c1dcece5ca:37717 started persisting bucket cache to file 2024-11-18T03:48:21,002 DEBUG [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-18T03:48:21,002 DEBUG [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-18T03:48:21,005 DEBUG [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-18T03:48:21,006 DEBUG [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731901701002 2024-11-18T03:48:21,006 DEBUG [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-18T03:48:21,006 DEBUG [RS:1;88c1dcece5ca:33811 {}] bucket.BucketCache(1509): Thread RS:1;88c1dcece5ca:33811 finished persisting bucket cache to file, renaming 2024-11-18T03:48:21,006 DEBUG [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731901701002 2024-11-18T03:48:21,006 DEBUG [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1509): Thread RS:2;88c1dcece5ca:37717 finished persisting bucket cache to file, renaming 2024-11-18T03:48:21,006 WARN [RS:2;88c1dcece5ca:37717 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-18T03:48:21,006 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T03:48:21,006 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-18T03:48:21,006 INFO [RS:1;88c1dcece5ca:33811 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T03:48:21,006 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T03:48:21,006 INFO [RS:1;88c1dcece5ca:33811 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T03:48:21,006 INFO [RS:2;88c1dcece5ca:37717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-18T03:48:21,006 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:21,006 INFO [RS:2;88c1dcece5ca:37717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-18T03:48:21,006 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-18T03:48:21,006 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(959): stopping server 88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:21,006 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:21,006 INFO [RS:2;88c1dcece5ca:37717 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;88c1dcece5ca:37717. 2024-11-18T03:48:21,006 DEBUG [RS:2;88c1dcece5ca:37717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:21,006 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(959): stopping server 88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:21,006 DEBUG [RS:2;88c1dcece5ca:37717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:21,006 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:21,006 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(976): stopping server 88c1dcece5ca,37717,1731901700398; all regions closed. 2024-11-18T03:48:21,006 INFO [RS:1;88c1dcece5ca:33811 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;88c1dcece5ca:33811. 2024-11-18T03:48:21,007 DEBUG [RS:1;88c1dcece5ca:33811 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-18T03:48:21,007 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T03:48:21,007 DEBUG [RS:1;88c1dcece5ca:33811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:21,007 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ff148b60f3b6f5b01616332f09576c8, disabling compactions & flushes 2024-11-18T03:48:21,007 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:21,007 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T03:48:21,007 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T03:48:21,007 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:21,007 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T03:48:21,007 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. after waiting 0 ms 2024-11-18T03:48:21,007 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-18T03:48:21,007 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:21,007 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-18T03:48:21,007 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 1ff148b60f3b6f5b01616332f09576c8=testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8.} 2024-11-18T03:48:21,007 DEBUG [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ff148b60f3b6f5b01616332f09576c8 2024-11-18T03:48:21,007 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-18T03:48:21,008 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-18T03:48:21,008 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-18T03:48:21,008 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-18T03:48:21,008 DEBUG [StoreFileCloser-1ff148b60f3b6f5b01616332f09576c8-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/family/192f69d2a8074f0cbd43aca485a9e4e0 2024-11-18T03:48:21,008 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-18T03:48:21,008 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-18T03:48:21,008 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-18T03:48:21,009 DEBUG [RS:2;88c1dcece5ca:37717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C37717%2C1731901700398:(num 1731901700589) 2024-11-18T03:48:21,009 DEBUG [RS:2;88c1dcece5ca:37717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.ChoreService(370): Chore service for: regionserver/88c1dcece5ca:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-18T03:48:21,009 INFO [regionserver/88c1dcece5ca:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:21,009 INFO [RS:2;88c1dcece5ca:37717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37717 2024-11-18T03:48:21,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88c1dcece5ca,37717,1731901700398 2024-11-18T03:48:21,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:21,011 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:21,013 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88c1dcece5ca,37717,1731901700398] 2024-11-18T03:48:21,013 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/default/testBlockEvictionOnGracefulStop1731901698201/1ff148b60f3b6f5b01616332f09576c8/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-18T03:48:21,014 INFO [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:21,014 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ff148b60f3b6f5b01616332f09576c8: Waiting for close lock at 1731901701006Running coprocessor pre-close hooks at 1731901701007 (+1 ms)Disabling compacts and flushes for region at 1731901701007Disabling writes for close at 1731901701007Writing region close event to WAL at 1731901701008 (+1 ms)Running coprocessor post-close hooks at 1731901701014 (+6 ms)Closed at 1731901701014 2024-11-18T03:48:21,014 DEBUG [RS_CLOSE_REGION-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8. 2024-11-18T03:48:21,014 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88c1dcece5ca,37717,1731901700398 already deleted, retry=false 2024-11-18T03:48:21,014 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88c1dcece5ca,37717,1731901700398 expired; onlineServers=1 2024-11-18T03:48:21,020 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 95c309d0f0094c229f9eb9112c88948b 2024-11-18T03:48:21,034 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/info/95c309d0f0094c229f9eb9112c88948b is 209, key is testBlockEvictionOnGracefulStop1731901698201,,1731901698205.1ff148b60f3b6f5b01616332f09576c8./info:regioninfo/1731901700445/Put/seqid=0 2024-11-18T03:48:21,035 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/info/95c309d0f0094c229f9eb9112c88948b 2024-11-18T03:48:21,042 INFO [regionserver/88c1dcece5ca:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:21,044 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 83b2a2eeca5f4521ab088555a7e8436a 2024-11-18T03:48:21,052 INFO [regionserver/88c1dcece5ca:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:21,058 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/ns/83b2a2eeca5f4521ab088555a7e8436a is 43, key is default/ns:d/1731901698171/Put/seqid=0 2024-11-18T03:48:21,059 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/ns/83b2a2eeca5f4521ab088555a7e8436a 2024-11-18T03:48:21,069 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on de8eabacb5ef46e1a53570387f3dfae6 2024-11-18T03:48:21,084 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/table/de8eabacb5ef46e1a53570387f3dfae6 is 80, key is testBlockEvictionOnGracefulStop1731901698201/table:state/1731901698580/Put/seqid=0 2024-11-18T03:48:21,084 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/table/de8eabacb5ef46e1a53570387f3dfae6 2024-11-18T03:48:21,086 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/info/95c309d0f0094c229f9eb9112c88948b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/95c309d0f0094c229f9eb9112c88948b 2024-11-18T03:48:21,088 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/95c309d0f0094c229f9eb9112c88948b, delay=936 ms 2024-11-18T03:48:21,089 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/95c309d0f0094c229f9eb9112c88948b, entries=14, sequenceid=13, filesize=7.5 K 2024-11-18T03:48:21,089 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/ns/83b2a2eeca5f4521ab088555a7e8436a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/83b2a2eeca5f4521ab088555a7e8436a 2024-11-18T03:48:21,090 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/83b2a2eeca5f4521ab088555a7e8436a, delay=943 ms 2024-11-18T03:48:21,091 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/83b2a2eeca5f4521ab088555a7e8436a, entries=2, sequenceid=13, filesize=5.0 K 2024-11-18T03:48:21,092 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/.tmp/table/de8eabacb5ef46e1a53570387f3dfae6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/de8eabacb5ef46e1a53570387f3dfae6 2024-11-18T03:48:21,094 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/de8eabacb5ef46e1a53570387f3dfae6, delay=918 ms 2024-11-18T03:48:21,095 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/de8eabacb5ef46e1a53570387f3dfae6, entries=2, sequenceid=13, filesize=5.3 K 2024-11-18T03:48:21,096 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false 2024-11-18T03:48:21,096 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/info/95c309d0f0094c229f9eb9112c88948b 2024-11-18T03:48:21,097 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/ns/83b2a2eeca5f4521ab088555a7e8436a 2024-11-18T03:48:21,097 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/table/de8eabacb5ef46e1a53570387f3dfae6 2024-11-18T03:48:21,102 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-18T03:48:21,102 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-18T03:48:21,102 INFO [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:21,102 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731901701007Running coprocessor pre-close hooks at 1731901701008 (+1 ms)Disabling compacts and flushes for region at 1731901701008Disabling writes for close at 1731901701008Obtaining lock to block concurrent updates at 1731901701008Preparing flush snapshotting stores in 1588230740 at 1731901701008Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731901701009 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731901701009Flushing 1588230740/info: creating writer at 1731901701009Flushing 1588230740/info: appending metadata at 1731901701034 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731901701034Flushing 1588230740/ns: creating writer at 1731901701038 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731901701058 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731901701058Flushing 1588230740/table: creating writer at 1731901701062 (+4 ms)Flushing 1588230740/table: appending metadata at 1731901701083 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731901701083Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66a11fc2: reopening flushed file at 1731901701086 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28b21d44: reopening flushed file at 1731901701089 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f3e1cc2: reopening flushed file at 1731901701092 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false at 1731901701096 (+4 ms)Writing region close event to WAL at 1731901701098 (+2 ms)Running coprocessor post-close hooks at 1731901701102 (+4 ms)Closed at 1731901701102 2024-11-18T03:48:21,103 DEBUG [RS_CLOSE_META-regionserver/88c1dcece5ca:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-18T03:48:21,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:21,113 INFO [RS:2;88c1dcece5ca:37717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:21,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37717-0x101293097240003, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:21,113 INFO [RS:2;88c1dcece5ca:37717 {}] regionserver.HRegionServer(1031): Exiting; stopping=88c1dcece5ca,37717,1731901700398; zookeeper connection closed. 2024-11-18T03:48:21,113 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5dc22e96 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5dc22e96 2024-11-18T03:48:21,208 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(976): stopping server 88c1dcece5ca,33811,1731901697268; all regions closed. 2024-11-18T03:48:21,210 DEBUG [RS:1;88c1dcece5ca:33811 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs 2024-11-18T03:48:21,210 INFO [RS:1;88c1dcece5ca:33811 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C33811%2C1731901697268.meta:.meta(num 1731901698122) 2024-11-18T03:48:21,212 DEBUG [RS:1;88c1dcece5ca:33811 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/oldWALs 2024-11-18T03:48:21,212 INFO [RS:1;88c1dcece5ca:33811 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 88c1dcece5ca%2C33811%2C1731901697268:(num 1731901697784) 2024-11-18T03:48:21,212 DEBUG [RS:1;88c1dcece5ca:33811 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-18T03:48:21,212 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.LeaseManager(133): Closed leases 2024-11-18T03:48:21,212 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:21,213 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.ChoreService(370): Chore service for: regionserver/88c1dcece5ca:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:21,213 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:21,213 INFO [regionserver/88c1dcece5ca:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:21,213 INFO [RS:1;88c1dcece5ca:33811 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33811 2024-11-18T03:48:21,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/88c1dcece5ca,33811,1731901697268 2024-11-18T03:48:21,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-18T03:48:21,217 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:21,218 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [88c1dcece5ca,33811,1731901697268] 2024-11-18T03:48:21,219 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/88c1dcece5ca,33811,1731901697268 already deleted, retry=false 2024-11-18T03:48:21,219 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 88c1dcece5ca,33811,1731901697268 expired; onlineServers=0 2024-11-18T03:48:21,219 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '88c1dcece5ca,43223,1731901697190' ***** 2024-11-18T03:48:21,219 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-18T03:48:21,219 INFO [M:0;88c1dcece5ca:43223 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-18T03:48:21,219 INFO [M:0;88c1dcece5ca:43223 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-18T03:48:21,220 DEBUG [M:0;88c1dcece5ca:43223 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-18T03:48:21,220 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-18T03:48:21,220 DEBUG [M:0;88c1dcece5ca:43223 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-18T03:48:21,220 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901697503 {}] cleaner.HFileCleaner(306): Exit Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.large.0-1731901697503,5,FailOnTimeoutGroup] 2024-11-18T03:48:21,220 DEBUG [master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901697503 {}] cleaner.HFileCleaner(306): Exit Thread[master/88c1dcece5ca:0:becomeActiveMaster-HFileCleaner.small.0-1731901697503,5,FailOnTimeoutGroup] 2024-11-18T03:48:21,220 INFO [M:0;88c1dcece5ca:43223 {}] hbase.ChoreService(370): Chore service for: master/88c1dcece5ca:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-18T03:48:21,220 INFO [M:0;88c1dcece5ca:43223 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-18T03:48:21,220 DEBUG [M:0;88c1dcece5ca:43223 {}] master.HMaster(1795): Stopping service threads 2024-11-18T03:48:21,220 INFO [M:0;88c1dcece5ca:43223 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-18T03:48:21,220 INFO [M:0;88c1dcece5ca:43223 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-18T03:48:21,221 INFO [M:0;88c1dcece5ca:43223 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-18T03:48:21,221 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-18T03:48:21,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-18T03:48:21,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-18T03:48:21,222 DEBUG [M:0;88c1dcece5ca:43223 {}] zookeeper.ZKUtil(347): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-18T03:48:21,222 WARN [M:0;88c1dcece5ca:43223 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-18T03:48:21,222 INFO [M:0;88c1dcece5ca:43223 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/.lastflushedseqids 2024-11-18T03:48:21,229 INFO [M:0;88c1dcece5ca:43223 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-18T03:48:21,229 INFO [M:0;88c1dcece5ca:43223 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-18T03:48:21,230 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-18T03:48:21,230 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:21,230 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:21,230 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-18T03:48:21,230 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:21,230 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-18T03:48:21,240 DEBUG [M:0;88c1dcece5ca:43223 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7edb0e9e04b24478b888ae9ec2c50f79 2024-11-18T03:48:21,254 DEBUG [M:0;88c1dcece5ca:43223 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7edb0e9e04b24478b888ae9ec2c50f79 is 82, key is hbase:meta,,1/info:regioninfo/1731901698156/Put/seqid=0 2024-11-18T03:48:21,254 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7edb0e9e04b24478b888ae9ec2c50f79 2024-11-18T03:48:21,262 DEBUG [M:0;88c1dcece5ca:43223 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7fd938ebedb345028965a1c5987ef93c 2024-11-18T03:48:21,276 DEBUG [M:0;88c1dcece5ca:43223 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7fd938ebedb345028965a1c5987ef93c is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731901698585/Put/seqid=0 2024-11-18T03:48:21,277 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7fd938ebedb345028965a1c5987ef93c 2024-11-18T03:48:21,285 DEBUG [M:0;88c1dcece5ca:43223 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eec2ccff72654175b95d931bdc9e5f61 2024-11-18T03:48:21,299 DEBUG [M:0;88c1dcece5ca:43223 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eec2ccff72654175b95d931bdc9e5f61 is 69, key is 88c1dcece5ca,33811,1731901697268/rs:state/1731901697615/Put/seqid=0 2024-11-18T03:48:21,299 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eec2ccff72654175b95d931bdc9e5f61 2024-11-18T03:48:21,301 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eec2ccff72654175b95d931bdc9e5f61 2024-11-18T03:48:21,302 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7edb0e9e04b24478b888ae9ec2c50f79 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7edb0e9e04b24478b888ae9ec2c50f79 2024-11-18T03:48:21,303 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7edb0e9e04b24478b888ae9ec2c50f79, entries=8, sequenceid=83, filesize=5.5 K 2024-11-18T03:48:21,304 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7fd938ebedb345028965a1c5987ef93c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7fd938ebedb345028965a1c5987ef93c 2024-11-18T03:48:21,305 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7fd938ebedb345028965a1c5987ef93c, entries=9, sequenceid=83, filesize=6.4 K 2024-11-18T03:48:21,306 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eec2ccff72654175b95d931bdc9e5f61 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eec2ccff72654175b95d931bdc9e5f61 2024-11-18T03:48:21,307 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for eec2ccff72654175b95d931bdc9e5f61 2024-11-18T03:48:21,307 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3b4ae25b-e8b1-5c39-25c5-a532daae83bd/54af5b4f-e9ff-c3a9-99e5-78a289d4be7a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eec2ccff72654175b95d931bdc9e5f61, entries=3, sequenceid=83, filesize=5.3 K 2024-11-18T03:48:21,308 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false 2024-11-18T03:48:21,310 INFO [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-18T03:48:21,310 DEBUG [M:0;88c1dcece5ca:43223 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731901701230Disabling compacts and flushes for region at 1731901701230Disabling writes for close at 1731901701230Obtaining lock to block concurrent updates at 1731901701230Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731901701230Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1731901701231 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731901701231Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731901701231Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731901701253 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731901701253Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731901701256 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731901701275 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731901701276 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731901701278 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731901701298 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731901701298Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cfb9dff: reopening flushed file at 1731901701301 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3454c84e: reopening flushed file at 1731901701304 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f240ddb: reopening flushed file at 1731901701305 (+1 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false at 1731901701308 (+3 ms)Writing region close event to WAL at 1731901701310 (+2 ms)Closed at 1731901701310 2024-11-18T03:48:21,311 INFO [M:0;88c1dcece5ca:43223 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-18T03:48:21,311 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-18T03:48:21,311 INFO [M:0;88c1dcece5ca:43223 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43223 2024-11-18T03:48:21,311 INFO [M:0;88c1dcece5ca:43223 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-18T03:48:21,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:21,318 INFO [RS:1;88c1dcece5ca:33811 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:21,318 INFO [RS:1;88c1dcece5ca:33811 {}] regionserver.HRegionServer(1031): Exiting; stopping=88c1dcece5ca,33811,1731901697268; zookeeper connection closed. 2024-11-18T03:48:21,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33811-0x101293097240002, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:21,318 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@578f7cb3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@578f7cb3 2024-11-18T03:48:21,319 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-18T03:48:21,415 INFO [M:0;88c1dcece5ca:43223 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-18T03:48:21,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:21,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43223-0x101293097240000, quorum=127.0.0.1:61427, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-18T03:48:21,418 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-18T03:48:21,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-18T03:48:21,419 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-18T03:48:21,426 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=609 (was 627), ProcessCount=11 (was 11), AvailableMemoryMB=5422 (was 5490)