2024-11-20 06:54:55,235 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 06:54:55,254 main DEBUG Took 0.011287 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 06:54:55,254 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 06:54:55,254 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 06:54:55,255 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 06:54:55,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,263 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 06:54:55,275 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,277 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,279 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,280 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,281 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,283 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,284 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,285 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,287 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,288 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,288 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,289 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,290 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 06:54:55,291 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,291 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 06:54:55,293 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 06:54:55,295 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 06:54:55,297 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 06:54:55,298 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 06:54:55,300 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 06:54:55,300 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 06:54:55,310 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 06:54:55,313 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 06:54:55,316 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 06:54:55,316 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 06:54:55,317 main DEBUG createAppenders(={Console}) 2024-11-20 06:54:55,318 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-20 06:54:55,318 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-20 06:54:55,319 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-20 06:54:55,319 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 06:54:55,319 main DEBUG OutputStream closed 2024-11-20 06:54:55,320 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 06:54:55,320 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 06:54:55,320 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-20 06:54:55,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 06:54:55,390 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 06:54:55,391 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 06:54:55,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 06:54:55,392 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 06:54:55,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 06:54:55,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 06:54:55,393 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 06:54:55,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 06:54:55,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 06:54:55,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 06:54:55,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 06:54:55,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 06:54:55,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 06:54:55,395 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 06:54:55,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 06:54:55,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 06:54:55,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 06:54:55,399 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 06:54:55,399 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-20 06:54:55,399 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 06:54:55,400 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-20T06:54:55,412 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-20 06:54:55,414 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 06:54:55,414 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T06:54:55,633 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3 2024-11-20T06:54:55,656 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=393, ProcessCount=11, AvailableMemoryMB=5107 2024-11-20T06:54:55,748 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T06:54:55,836 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:55,851 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/cluster_577fd111-48f4-b14a-8add-e194b3095eb6, deleteOnExit=true 2024-11-20T06:54:55,979 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/cluster_577fd111-48f4-b14a-8add-e194b3095eb6/zookeeper_0, clientPort=54382, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/cluster_577fd111-48f4-b14a-8add-e194b3095eb6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/cluster_577fd111-48f4-b14a-8add-e194b3095eb6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T06:54:55,989 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54382 2024-11-20T06:54:55,990 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:56,066 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 with version=8 2024-11-20T06:54:56,067 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/hbase-staging 2024-11-20T06:54:56,159 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T06:54:56,442 INFO [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:54:56,451 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:56,452 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:56,462 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:54:56,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:56,463 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:54:56,602 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T06:54:56,686 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:54:56,732 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T06:54:56,742 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T06:54:56,745 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:54:56,784 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 96437 (auto-detected) 2024-11-20T06:54:56,785 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-20T06:54:56,808 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34697 2024-11-20T06:54:56,828 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34697 connecting to ZooKeeper ensemble=127.0.0.1:54382 2024-11-20T06:54:56,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:346970x0, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:54:56,947 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34697-0x101584fd8990000 connected 2024-11-20T06:54:57,042 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:57,044 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:57,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:54:57,057 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63, hbase.cluster.distributed=false 2024-11-20T06:54:57,095 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:54:57,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34697 2024-11-20T06:54:57,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34697 2024-11-20T06:54:57,101 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34697 2024-11-20T06:54:57,102 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34697 2024-11-20T06:54:57,102 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34697 2024-11-20T06:54:57,179 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:54:57,181 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:57,181 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:57,182 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:54:57,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:57,182 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:54:57,184 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T06:54:57,187 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:54:57,188 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45557 2024-11-20T06:54:57,190 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45557 connecting to ZooKeeper ensemble=127.0.0.1:54382 2024-11-20T06:54:57,191 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:57,192 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:57,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:455570x0, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:54:57,231 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:455570x0, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:54:57,232 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45557-0x101584fd8990001 connected 2024-11-20T06:54:57,241 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T06:54:57,253 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.cache 2024-11-20T06:54:57,258 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T06:54:57,259 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T06:54:57,261 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T06:54:57,262 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T06:54:57,262 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T06:54:57,267 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T06:54:57,272 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T06:54:57,272 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T06:54:57,272 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.persistence is complete. 2024-11-20T06:54:57,275 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T06:54:57,280 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:54:57,281 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45557 2024-11-20T06:54:57,282 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45557 2024-11-20T06:54:57,283 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45557 2024-11-20T06:54:57,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45557 2024-11-20T06:54:57,284 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45557 2024-11-20T06:54:57,307 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:54:57,307 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:57,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:57,308 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:54:57,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:54:57,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:54:57,309 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T06:54:57,309 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:54:57,310 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42993 2024-11-20T06:54:57,313 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42993 connecting to ZooKeeper ensemble=127.0.0.1:54382 2024-11-20T06:54:57,313 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:57,314 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:57,343 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429930x0, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:54:57,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429930x0, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:54:57,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42993-0x101584fd8990002 connected 2024-11-20T06:54:57,345 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T06:54:57,346 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.cache 2024-11-20T06:54:57,346 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T06:54:57,346 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T06:54:57,347 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T06:54:57,349 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T06:54:57,349 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T06:54:57,349 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T06:54:57,350 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T06:54:57,350 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T06:54:57,350 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.persistence is complete. 2024-11-20T06:54:57,351 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T06:54:57,354 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:54:57,354 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42993 2024-11-20T06:54:57,358 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42993 2024-11-20T06:54:57,359 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42993 2024-11-20T06:54:57,360 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42993 2024-11-20T06:54:57,361 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42993 2024-11-20T06:54:57,382 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:34697 2024-11-20T06:54:57,383 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:57,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:54:57,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:54:57,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:54:57,400 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:57,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T06:54:57,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T06:54:57,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:57,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:57,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:57,432 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T06:54:57,433 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,34697,1732085696275 from backup master directory 2024-11-20T06:54:57,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:54:57,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:57,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:54:57,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:54:57,444 WARN [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:54:57,445 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:57,447 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T06:54:57,448 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T06:54:57,456 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/hbase.id] with ID: a7a4a913-d216-4dbc-88aa-7f1007e89e7c 2024-11-20T06:54:57,456 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.tmp/hbase.id 2024-11-20T06:54:57,461 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/hbase.id] 2024-11-20T06:54:57,495 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:54:57,502 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T06:54:57,527 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-20T06:54:57,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:57,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:57,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:57,588 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:54:57,593 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T06:54:57,633 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:54:57,639 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:54:57,687 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store 2024-11-20T06:54:57,707 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T06:54:57,710 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:54:57,711 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T06:54:57,712 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:54:57,712 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:54:57,713 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T06:54:57,714 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:54:57,714 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:54:57,714 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732085697711Disabling compacts and flushes for region at 1732085697711Disabling writes for close at 1732085697714 (+3 ms)Writing region close event to WAL at 1732085697714Closed at 1732085697714 2024-11-20T06:54:57,715 WARN [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/.initializing 2024-11-20T06:54:57,715 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/WALs/61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:57,729 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:54:57,743 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C34697%2C1732085696275, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/WALs/61c2066a8a0f,34697,1732085696275, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/oldWALs, maxLogs=10 2024-11-20T06:54:57,779 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/WALs/61c2066a8a0f,34697,1732085696275/61c2066a8a0f%2C34697%2C1732085696275.1732085697747 2024-11-20T06:54:57,780 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:54:57,781 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:54:57,781 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:54:57,785 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,786 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,870 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T06:54:57,874 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:57,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:57,876 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T06:54:57,881 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:57,881 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:54:57,882 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T06:54:57,886 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:57,886 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:54:57,887 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,891 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T06:54:57,891 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:57,892 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:54:57,892 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,897 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,898 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,903 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,904 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,908 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:54:57,912 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:54:57,920 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:54:57,921 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59717868, jitterRate=-0.11013442277908325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:54:57,932 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732085697798Initializing all the Stores at 1732085697799 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085697799Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085697800 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085697800Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085697801 (+1 ms)Cleaning up temporary data from old regions at 1732085697904 (+103 ms)Region opened successfully at 1732085697932 (+28 ms) 2024-11-20T06:54:57,934 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T06:54:57,970 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c9b52c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:54:58,001 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T06:54:58,014 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T06:54:58,014 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T06:54:58,017 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T06:54:58,018 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T06:54:58,023 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T06:54:58,023 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T06:54:58,047 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T06:54:58,056 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T06:54:58,110 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T06:54:58,112 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T06:54:58,114 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T06:54:58,121 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T06:54:58,124 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T06:54:58,129 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T06:54:58,138 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T06:54:58,140 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T06:54:58,151 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T06:54:58,169 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T06:54:58,180 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T06:54:58,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:54:58,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:54:58,201 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:54:58,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,206 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,34697,1732085696275, sessionid=0x101584fd8990000, setting cluster-up flag (Was=false) 2024-11-20T06:54:58,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,263 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T06:54:58,266 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:58,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:58,321 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T06:54:58,325 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:58,331 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T06:54:58,366 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(746): ClusterId : a7a4a913-d216-4dbc-88aa-7f1007e89e7c 2024-11-20T06:54:58,366 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(746): ClusterId : a7a4a913-d216-4dbc-88aa-7f1007e89e7c 2024-11-20T06:54:58,368 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T06:54:58,368 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T06:54:58,387 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T06:54:58,387 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T06:54:58,387 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T06:54:58,387 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T06:54:58,398 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T06:54:58,398 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T06:54:58,398 DEBUG [RS:0;61c2066a8a0f:45557 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156ca54a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:54:58,399 DEBUG [RS:1;61c2066a8a0f:42993 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@199c239c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:54:58,409 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T06:54:58,413 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;61c2066a8a0f:42993 2024-11-20T06:54:58,415 DEBUG [RS:0;61c2066a8a0f:45557 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:45557 2024-11-20T06:54:58,418 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T06:54:58,418 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T06:54:58,418 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T06:54:58,418 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T06:54:58,418 DEBUG [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T06:54:58,418 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T06:54:58,421 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,34697,1732085696275 with port=45557, startcode=1732085697147 2024-11-20T06:54:58,423 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,34697,1732085696275 with port=42993, startcode=1732085697306 2024-11-20T06:54:58,424 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T06:54:58,433 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T06:54:58,434 DEBUG [RS:1;61c2066a8a0f:42993 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T06:54:58,434 DEBUG [RS:0;61c2066a8a0f:45557 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T06:54:58,440 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,34697,1732085696275 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T06:54:58,449 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:54:58,450 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:54:58,450 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:54:58,450 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:54:58,450 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10 2024-11-20T06:54:58,451 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,451 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:54:58,451 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,461 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085728461 2024-11-20T06:54:58,462 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:54:58,462 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T06:54:58,464 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T06:54:58,465 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T06:54:58,468 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:58,468 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T06:54:58,469 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T06:54:58,468 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T06:54:58,469 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T06:54:58,469 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T06:54:58,474 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,483 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T06:54:58,485 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T06:54:58,485 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T06:54:58,491 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T06:54:58,491 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T06:54:58,498 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085698492,5,FailOnTimeoutGroup] 2024-11-20T06:54:58,504 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085698499,5,FailOnTimeoutGroup] 2024-11-20T06:54:58,504 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,504 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T06:54:58,505 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,506 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,510 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T06:54:58,511 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33341, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T06:54:58,511 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59999, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T06:54:58,511 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:54:58,518 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34697 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:58,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:54:58,521 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T06:54:58,521 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34697 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:58,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T06:54:58,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:58,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:58,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T06:54:58,533 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34697 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,45557,1732085697147 2024-11-20T06:54:58,534 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34697 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,45557,1732085697147 2024-11-20T06:54:58,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T06:54:58,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:58,536 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:58,537 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T06:54:58,538 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:54:58,538 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T06:54:58,538 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T06:54:58,539 DEBUG [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:54:58,539 DEBUG [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T06:54:58,539 DEBUG [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T06:54:58,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T06:54:58,542 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:58,547 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:58,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T06:54:58,552 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T06:54:58,552 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:58,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:58,553 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T06:54:58,554 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:54:58,555 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:54:58,555 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T06:54:58,555 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T06:54:58,556 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:54:58,557 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T06:54:58,564 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:54:58,565 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58855773, jitterRate=-0.12298063933849335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732085698520Initializing all the Stores at 1732085698520Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085698520Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085698521 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085698521Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085698521Cleaning up temporary data from old regions at 1732085698555 (+34 ms)Region opened successfully at 1732085698570 (+15 ms) 2024-11-20T06:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T06:54:58,570 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T06:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T06:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T06:54:58,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T06:54:58,572 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T06:54:58,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732085698570Disabling compacts and flushes for region at 1732085698570Disabling writes for close at 1732085698570Writing region close event to WAL at 1732085698571 (+1 ms)Closed at 1732085698572 (+1 ms) 2024-11-20T06:54:58,574 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:54:58,574 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T06:54:58,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T06:54:58,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:54:58,587 DEBUG [RS:0;61c2066a8a0f:45557 {}] zookeeper.ZKUtil(111): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,45557,1732085697147 2024-11-20T06:54:58,587 DEBUG [RS:1;61c2066a8a0f:42993 {}] zookeeper.ZKUtil(111): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:58,587 WARN [RS:0;61c2066a8a0f:45557 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:54:58,587 WARN [RS:1;61c2066a8a0f:42993 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:54:58,587 INFO [RS:0;61c2066a8a0f:45557 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:54:58,587 INFO [RS:1;61c2066a8a0f:42993 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:54:58,587 DEBUG [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,45557,1732085697147 2024-11-20T06:54:58,587 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:58,592 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,42993,1732085697306] 2024-11-20T06:54:58,593 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,45557,1732085697147] 2024-11-20T06:54:58,593 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:54:58,595 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T06:54:58,617 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T06:54:58,617 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T06:54:58,640 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T06:54:58,640 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T06:54:58,645 INFO [RS:1;61c2066a8a0f:42993 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T06:54:58,645 INFO [RS:0;61c2066a8a0f:45557 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T06:54:58,645 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,645 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,646 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T06:54:58,646 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T06:54:58,651 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T06:54:58,651 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T06:54:58,652 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,652 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,653 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,653 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:54:58,654 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,654 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:54:58,654 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,655 DEBUG [RS:1;61c2066a8a0f:42993 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:54:58,655 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,655 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:54:58,655 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:54:58,655 DEBUG [RS:0;61c2066a8a0f:45557 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:54:58,656 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,656 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42993,1732085697306-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:54:58,657 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,45557,1732085697147-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:54:58,677 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T06:54:58,677 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T06:54:58,680 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,45557,1732085697147-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,680 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,42993,1732085697306-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,680 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,680 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,680 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.Replication(171): 61c2066a8a0f,45557,1732085697147 started 2024-11-20T06:54:58,680 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.Replication(171): 61c2066a8a0f,42993,1732085697306 started 2024-11-20T06:54:58,698 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,698 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:58,699 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,45557,1732085697147, RpcServer on 61c2066a8a0f/172.17.0.2:45557, sessionid=0x101584fd8990001 2024-11-20T06:54:58,699 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,42993,1732085697306, RpcServer on 61c2066a8a0f/172.17.0.2:42993, sessionid=0x101584fd8990002 2024-11-20T06:54:58,700 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T06:54:58,700 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T06:54:58,700 DEBUG [RS:0;61c2066a8a0f:45557 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,45557,1732085697147 2024-11-20T06:54:58,700 DEBUG [RS:1;61c2066a8a0f:42993 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:58,700 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,45557,1732085697147' 2024-11-20T06:54:58,700 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,42993,1732085697306' 2024-11-20T06:54:58,700 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T06:54:58,700 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T06:54:58,701 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T06:54:58,701 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T06:54:58,702 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T06:54:58,702 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T06:54:58,702 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T06:54:58,702 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T06:54:58,702 DEBUG [RS:1;61c2066a8a0f:42993 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:58,702 DEBUG [RS:0;61c2066a8a0f:45557 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,45557,1732085697147 2024-11-20T06:54:58,702 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,42993,1732085697306' 2024-11-20T06:54:58,702 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,45557,1732085697147' 2024-11-20T06:54:58,702 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T06:54:58,702 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T06:54:58,702 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T06:54:58,703 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T06:54:58,703 DEBUG [RS:1;61c2066a8a0f:42993 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T06:54:58,703 INFO [RS:1;61c2066a8a0f:42993 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T06:54:58,703 INFO [RS:1;61c2066a8a0f:42993 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T06:54:58,703 DEBUG [RS:0;61c2066a8a0f:45557 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T06:54:58,703 INFO [RS:0;61c2066a8a0f:45557 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T06:54:58,703 INFO [RS:0;61c2066a8a0f:45557 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T06:54:58,746 WARN [61c2066a8a0f:34697 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T06:54:58,810 INFO [RS:1;61c2066a8a0f:42993 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:54:58,810 INFO [RS:0;61c2066a8a0f:45557 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:54:58,812 INFO [RS:1;61c2066a8a0f:42993 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C42993%2C1732085697306, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,42993,1732085697306, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:54:58,812 INFO [RS:0;61c2066a8a0f:45557 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C45557%2C1732085697147, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,45557,1732085697147, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:54:58,834 INFO [RS:1;61c2066a8a0f:42993 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,42993,1732085697306/61c2066a8a0f%2C42993%2C1732085697306.1732085698817 2024-11-20T06:54:58,834 INFO [RS:0;61c2066a8a0f:45557 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,45557,1732085697147/61c2066a8a0f%2C45557%2C1732085697147.1732085698817 2024-11-20T06:54:58,835 DEBUG [RS:0;61c2066a8a0f:45557 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:54:58,835 DEBUG [RS:1;61c2066a8a0f:42993 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:54:58,999 DEBUG [61c2066a8a0f:34697 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T06:54:59,010 DEBUG [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(204): Hosts are {61c2066a8a0f=0} racks are {/default-rack=0} 2024-11-20T06:54:59,013 DEBUG [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T06:54:59,013 DEBUG [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T06:54:59,013 DEBUG [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T06:54:59,013 DEBUG [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T06:54:59,013 INFO [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T06:54:59,013 INFO [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T06:54:59,014 DEBUG [61c2066a8a0f:34697 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T06:54:59,020 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:59,025 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,42993,1732085697306, state=OPENING 2024-11-20T06:54:59,092 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T06:54:59,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:59,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:59,136 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:54:59,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:54:59,138 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:54:59,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:54:59,141 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:54:59,144 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,42993,1732085697306}] 2024-11-20T06:54:59,318 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T06:54:59,321 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40261, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T06:54:59,329 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T06:54:59,329 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:54:59,330 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T06:54:59,331 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C42993%2C1732085697306.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,42993,1732085697306, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:54:59,352 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,42993,1732085697306/61c2066a8a0f%2C42993%2C1732085697306.meta.1732085699333.meta 2024-11-20T06:54:59,353 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:54:59,353 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:54:59,355 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T06:54:59,357 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T06:54:59,361 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T06:54:59,365 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T06:54:59,366 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:54:59,366 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T06:54:59,366 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T06:54:59,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T06:54:59,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T06:54:59,369 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:59,369 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:59,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T06:54:59,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T06:54:59,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:59,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:59,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T06:54:59,371 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T06:54:59,372 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:59,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:59,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T06:54:59,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T06:54:59,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:59,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:54:59,373 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T06:54:59,374 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:54:59,375 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:54:59,376 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T06:54:59,376 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T06:54:59,376 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:54:59,377 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T06:54:59,377 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60019868, jitterRate=-0.10563427209854126}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:54:59,377 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T06:54:59,379 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732085699367Writing region info on filesystem at 1732085699367Initializing all the Stores at 1732085699367Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085699367Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085699368 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085699368Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085699368Cleaning up temporary data from old regions at 1732085699376 (+8 ms)Running coprocessor post-open hooks at 1732085699378 (+2 ms)Region opened successfully at 1732085699379 (+1 ms) 2024-11-20T06:54:59,384 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085699310 2024-11-20T06:54:59,395 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T06:54:59,395 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T06:54:59,396 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:59,397 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,42993,1732085697306, state=OPEN 2024-11-20T06:54:59,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:54:59,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:54:59,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:54:59,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:54:59,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:54:59,418 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:54:59,419 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,42993,1732085697306 2024-11-20T06:54:59,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T06:54:59,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,42993,1732085697306 in 275 msec 2024-11-20T06:54:59,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T06:54:59,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 842 msec 2024-11-20T06:54:59,428 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:54:59,428 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T06:54:59,445 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:54:59,446 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,42993,1732085697306, seqNum=-1] 2024-11-20T06:54:59,463 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:54:59,466 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44897, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:54:59,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1240 sec 2024-11-20T06:54:59,488 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085699487, completionTime=-1 2024-11-20T06:54:59,490 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T06:54:59,490 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T06:54:59,512 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T06:54:59,512 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085759512 2024-11-20T06:54:59,513 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085819513 2024-11-20T06:54:59,513 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 22 msec 2024-11-20T06:54:59,515 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T06:54:59,520 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34697,1732085696275-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:59,520 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34697,1732085696275-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:59,521 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34697,1732085696275-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:59,522 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:34697, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:59,522 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:59,523 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:59,528 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T06:54:59,548 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.103sec 2024-11-20T06:54:59,549 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T06:54:59,550 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T06:54:59,551 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T06:54:59,551 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T06:54:59,551 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T06:54:59,552 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34697,1732085696275-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:54:59,552 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34697,1732085696275-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T06:54:59,553 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T06:54:59,554 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T06:54:59,554 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,34697,1732085696275-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:54:59,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:54:59,578 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T06:54:59,578 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T06:54:59,582 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,34697,-1 for getting cluster id 2024-11-20T06:54:59,584 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:54:59,593 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a7a4a913-d216-4dbc-88aa-7f1007e89e7c' 2024-11-20T06:54:59,595 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:54:59,595 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a7a4a913-d216-4dbc-88aa-7f1007e89e7c" 2024-11-20T06:54:59,596 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4741bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:54:59,596 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,34697,-1] 2024-11-20T06:54:59,598 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:54:59,600 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:54:59,601 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:54:59,604 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:54:59,605 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:54:59,613 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,42993,1732085697306, seqNum=-1] 2024-11-20T06:54:59,613 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:54:59,615 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:54:59,642 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:59,646 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:54:59,653 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,34697,1732085696275 2024-11-20T06:54:59,657 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4028e783 2024-11-20T06:54:59,658 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:54:59,660 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:54:59,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732085699642', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:54:59,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 2024-11-20T06:54:59,674 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:54:59,676 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732085699642" procId is: 4 2024-11-20T06:54:59,676 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:54:59,677 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:54:59,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:54:59,691 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6965edae25549910ad73492f24200332, NAME => 'testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732085699642', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:54:59,697 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:54:59,698 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(1722): Closing 6965edae25549910ad73492f24200332, disabling compactions & flushes 2024-11-20T06:54:59,698 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:54:59,698 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:54:59,698 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. after waiting 0 ms 2024-11-20T06:54:59,698 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:54:59,698 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:54:59,698 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732085699642-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6965edae25549910ad73492f24200332: Waiting for close lock at 1732085699698Disabling compacts and flushes for region at 1732085699698Disabling writes for close at 1732085699698Writing region close event to WAL at 1732085699698Closed at 1732085699698 2024-11-20T06:54:59,699 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:54:59,704 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732085699700"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085699700"}]},"ts":"1732085699700"} 2024-11-20T06:54:59,707 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:54:59,708 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:54:59,711 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732085699642","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085699709"}]},"ts":"1732085699709"} 2024-11-20T06:54:59,714 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732085699642, state=ENABLING in hbase:meta 2024-11-20T06:54:59,715 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {61c2066a8a0f=0} racks are {/default-rack=0} 2024-11-20T06:54:59,715 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T06:54:59,715 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T06:54:59,715 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T06:54:59,715 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T06:54:59,715 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T06:54:59,715 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T06:54:59,716 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T06:54:59,717 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, ASSIGN}] 2024-11-20T06:54:59,718 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, ASSIGN 2024-11-20T06:54:59,719 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, ASSIGN; state=OFFLINE, location=61c2066a8a0f,45557,1732085697147; forceNewPlan=false, retain=false 2024-11-20T06:54:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:54:59,873 INFO [61c2066a8a0f:34697 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T06:54:59,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6965edae25549910ad73492f24200332, regionState=OPENING, regionLocation=61c2066a8a0f,45557,1732085697147 2024-11-20T06:54:59,878 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, ASSIGN because future has completed 2024-11-20T06:54:59,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,45557,1732085697147}] 2024-11-20T06:54:59,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:55:00,034 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T06:55:00,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35555, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T06:55:00,039 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:00,039 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6965edae25549910ad73492f24200332, NAME => 'testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:00,040 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732085699642 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,040 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:00,041 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,041 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,042 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,049 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6965edae25549910ad73492f24200332 columnFamilyName family 2024-11-20T06:55:00,049 DEBUG [StoreOpener-6965edae25549910ad73492f24200332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:00,050 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.HStore(327): Store=6965edae25549910ad73492f24200332/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:00,050 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,051 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332 2024-11-20T06:55:00,051 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332 2024-11-20T06:55:00,052 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,052 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,052 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,058 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:55:00,059 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6965edae25549910ad73492f24200332; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74248947, jitterRate=0.10639552772045135}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T06:55:00,059 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6965edae25549910ad73492f24200332 2024-11-20T06:55:00,060 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6965edae25549910ad73492f24200332: Running coprocessor pre-open hook at 1732085700041Writing region info on filesystem at 1732085700041Initializing all the Stores at 1732085700042 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085700042Cleaning up temporary data from old regions at 1732085700052 (+10 ms)Running coprocessor post-open hooks at 1732085700059 (+7 ms)Region opened successfully at 1732085700059 2024-11-20T06:55:00,061 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332., pid=6, masterSystemTime=1732085700033 2024-11-20T06:55:00,064 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:00,064 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:00,064 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6965edae25549910ad73492f24200332, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,45557,1732085697147 2024-11-20T06:55:00,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,45557,1732085697147 because future has completed 2024-11-20T06:55:00,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T06:55:00,071 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,45557,1732085697147 in 190 msec 2024-11-20T06:55:00,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T06:55:00,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, ASSIGN in 354 msec 2024-11-20T06:55:00,074 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:55:00,075 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732085699642","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085700074"}]},"ts":"1732085700074"} 2024-11-20T06:55:00,077 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732085699642, state=ENABLED in hbase:meta 2024-11-20T06:55:00,078 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:55:00,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732085699642 in 410 msec 2024-11-20T06:55:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:55:00,312 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732085699642 completed 2024-11-20T06:55:00,312 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732085699642 get assigned. Timeout = 60000ms 2024-11-20T06:55:00,317 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T06:55:00,327 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732085699642 assigned to meta. Checking AM states. 2024-11-20T06:55:00,328 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T06:55:00,329 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732085699642 assigned. 2024-11-20T06:55:00,336 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732085699642', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332., hostname=61c2066a8a0f,45557,1732085697147, seqNum=2] 2024-11-20T06:55:00,338 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:55:00,340 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:55:00,348 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6965edae25549910ad73492f24200332 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T06:55:00,371 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:00,407 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/.tmp/family/d322894e9a0d4a599d2a01270075492c is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T06:55:00,411 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/.tmp/family/d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:00,443 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/.tmp/family/d322894e9a0d4a599d2a01270075492c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:00,444 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c, delay=961 ms 2024-11-20T06:55:00,446 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T06:55:00,451 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 6965edae25549910ad73492f24200332 in 102ms, sequenceid=6, compaction requested=false 2024-11-20T06:55:00,452 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732085699642' 2024-11-20T06:55:00,454 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 6965edae25549910ad73492f24200332: 2024-11-20T06:55:01,408 DEBUG [hfile-prefetch-1732085700444 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d322894e9a0d4a599d2a01270075492c_0. 2024-11-20T06:55:01,413 DEBUG [hfile-prefetch-1732085700444 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d322894e9a0d4a599d2a01270075492c_117. 2024-11-20T06:55:01,414 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c, with total blocks 2, and data blocks 1 2024-11-20T06:55:01,414 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T06:55:01,415 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T06:55:01,415 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c. Will sleep 100ms and try the verification again. 2024-11-20T06:55:01,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=6965edae25549910ad73492f24200332, source=61c2066a8a0f,45557,1732085697147, destination=61c2066a8a0f,42993,1732085697306, warming up region on 61c2066a8a0f,42993,1732085697306 2024-11-20T06:55:01,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=6965edae25549910ad73492f24200332, source=61c2066a8a0f,45557,1732085697147, destination=61c2066a8a0f,42993,1732085697306, running balancer 2024-11-20T06:55:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, REOPEN/MOVE 2024-11-20T06:55:01,471 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, REOPEN/MOVE 2024-11-20T06:55:01,472 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(7855): Warmup {ENCODED => 6965edae25549910ad73492f24200332, NAME => 'testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:01,473 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6965edae25549910ad73492f24200332, regionState=CLOSING, regionLocation=61c2066a8a0f,45557,1732085697147 2024-11-20T06:55:01,473 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,474 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6965edae25549910ad73492f24200332 columnFamilyName family 2024-11-20T06:55:01,474 DEBUG [StoreOpener-6965edae25549910ad73492f24200332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:01,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, REOPEN/MOVE because future has completed 2024-11-20T06:55:01,477 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-20T06:55:01,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,45557,1732085697147}] 2024-11-20T06:55:01,479 DEBUG [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:01,480 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.HStore(327): Store=6965edae25549910ad73492f24200332/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(1722): Closing 6965edae25549910ad73492f24200332, disabling compactions & flushes 2024-11-20T06:55:01,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. after waiting 0 ms 2024-11-20T06:55:01,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,482 DEBUG [StoreFileCloser-6965edae25549910ad73492f24200332-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:01,484 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42993 {}] regionserver.HRegion(1676): Region close journal for 6965edae25549910ad73492f24200332: Waiting for close lock at 1732085701480Disabling compacts and flushes for region at 1732085701480Disabling writes for close at 1732085701481 (+1 ms)Writing region close event to WAL at 1732085701484 (+3 ms)Closed at 1732085701484 2024-11-20T06:55:01,485 WARN [hfile-prefetch-1732085700444 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-20T06:55:01,485 DEBUG [hfile-prefetch-1732085700444 {}] hfile.PrefetchExecutor(121): Prefetch completed for d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:01,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34697 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-20T06:55:01,552 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T06:55:01,552 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T06:55:01,553 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:01,553 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:01,565 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T06:55:01,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T06:55:01,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:01,565 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:01,571 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:01,571 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085701552 2024-11-20T06:55:01,571 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T06:55:01,573 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-20T06:55:01,574 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732085701565 2024-11-20T06:55:01,574 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T06:55:01,638 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,638 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-20T06:55:01,640 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 6965edae25549910ad73492f24200332, disabling compactions & flushes 2024-11-20T06:55:01,640 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,640 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,641 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. after waiting 0 ms 2024-11-20T06:55:01,641 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,645 DEBUG [StoreFileCloser-6965edae25549910ad73492f24200332-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file d322894e9a0d4a599d2a01270075492c, starting offset: 0, end offset: 9223372036854775807 2024-11-20T06:55:01,647 DEBUG [StoreFileCloser-6965edae25549910ad73492f24200332-family-1 {}] bucket.BucketCache(856): removed key d322894e9a0d4a599d2a01270075492c_0 from back map with offset lock 2094080 in the evict process 2024-11-20T06:55:01,653 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T06:55:01,655 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,655 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 6965edae25549910ad73492f24200332: Waiting for close lock at 1732085701640Running coprocessor pre-close hooks at 1732085701640Disabling compacts and flushes for region at 1732085701640Disabling writes for close at 1732085701641 (+1 ms)Writing region close event to WAL at 1732085701648 (+7 ms)Running coprocessor post-close hooks at 1732085701654 (+6 ms)Closed at 1732085701655 (+1 ms) 2024-11-20T06:55:01,655 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 6965edae25549910ad73492f24200332 move to 61c2066a8a0f,42993,1732085697306 record at close sequenceid=6 2024-11-20T06:55:01,658 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,658 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6965edae25549910ad73492f24200332, regionState=CLOSED 2024-11-20T06:55:01,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,45557,1732085697147 because future has completed 2024-11-20T06:55:01,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T06:55:01,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,45557,1732085697147 in 185 msec 2024-11-20T06:55:01,665 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, REOPEN/MOVE; state=CLOSED, location=61c2066a8a0f,42993,1732085697306; forceNewPlan=false, retain=false 2024-11-20T06:55:01,671 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T06:55:01,671 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T06:55:01,671 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:01,671 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:01,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:01,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085701671 2024-11-20T06:55:01,674 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T06:55:01,816 INFO [61c2066a8a0f:34697 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T06:55:01,816 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6965edae25549910ad73492f24200332, regionState=OPENING, regionLocation=61c2066a8a0f,42993,1732085697306 2024-11-20T06:55:01,820 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, REOPEN/MOVE because future has completed 2024-11-20T06:55:01,821 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,42993,1732085697306}] 2024-11-20T06:55:01,980 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:01,981 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 6965edae25549910ad73492f24200332, NAME => 'testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:01,983 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732085699642 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,983 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:01,984 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,984 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,987 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,988 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6965edae25549910ad73492f24200332 columnFamilyName family 2024-11-20T06:55:01,988 DEBUG [StoreOpener-6965edae25549910ad73492f24200332-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:01,990 DEBUG [StoreFileOpener-6965edae25549910ad73492f24200332-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c, delay=932 ms 2024-11-20T06:55:01,992 DEBUG [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:01,992 INFO [StoreOpener-6965edae25549910ad73492f24200332-1 {}] regionserver.HStore(327): Store=6965edae25549910ad73492f24200332/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:01,992 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,993 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332 2024-11-20T06:55:01,993 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332 2024-11-20T06:55:01,994 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,994 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,998 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,998 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 6965edae25549910ad73492f24200332; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63727720, jitterRate=-0.05038297176361084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T06:55:01,998 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6965edae25549910ad73492f24200332 2024-11-20T06:55:01,999 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 6965edae25549910ad73492f24200332: Running coprocessor pre-open hook at 1732085701985Writing region info on filesystem at 1732085701985Initializing all the Stores at 1732085701986 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085701986Cleaning up temporary data from old regions at 1732085701994 (+8 ms)Running coprocessor post-open hooks at 1732085701998 (+4 ms)Region opened successfully at 1732085701999 (+1 ms) 2024-11-20T06:55:02,000 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332., pid=9, masterSystemTime=1732085701974 2024-11-20T06:55:02,003 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:02,003 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:02,003 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=6965edae25549910ad73492f24200332, regionState=OPEN, openSeqNum=10, regionLocation=61c2066a8a0f,42993,1732085697306 2024-11-20T06:55:02,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,42993,1732085697306 because future has completed 2024-11-20T06:55:02,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-20T06:55:02,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 6965edae25549910ad73492f24200332, server=61c2066a8a0f,42993,1732085697306 in 186 msec 2024-11-20T06:55:02,011 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732085699642, region=6965edae25549910ad73492f24200332, REOPEN/MOVE in 539 msec 2024-11-20T06:55:02,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T06:55:02,024 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:55:02,024 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:02,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:02,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:02,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:55:02,028 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T06:55:02,029 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-11-20T06:55:02,029 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,34697,1732085696275 2024-11-20T06:55:02,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:02,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:02,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:02,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,059 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T06:55:02,060 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:55:02,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:02,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:02,060 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:02,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:02,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:02,061 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,45557,1732085697147' ***** 2024-11-20T06:55:02,061 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T06:55:02,061 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,42993,1732085697306' ***** 2024-11-20T06:55:02,061 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T06:55:02,061 INFO [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T06:55:02,062 INFO [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T06:55:02,062 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T06:55:02,062 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T06:55:02,062 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T06:55:02,062 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T06:55:02,062 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T06:55:02,062 INFO [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.persistence 2024-11-20T06:55:02,062 INFO [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T06:55:02,062 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T06:55:02,062 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T06:55:02,062 INFO [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/bucket.persistence 2024-11-20T06:55:02,062 INFO [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T06:55:02,063 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T06:55:02,072 DEBUG [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1487): Thread RS:1;61c2066a8a0f:42993 started persisting bucket cache to file 2024-11-20T06:55:02,072 DEBUG [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:02,072 DEBUG [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:02,073 DEBUG [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1487): Thread RS:0;61c2066a8a0f:45557 started persisting bucket cache to file 2024-11-20T06:55:02,073 DEBUG [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:02,073 DEBUG [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:02,075 DEBUG [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:02,075 DEBUG [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:02,076 DEBUG [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085702072 2024-11-20T06:55:02,076 DEBUG [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085702073 2024-11-20T06:55:02,076 DEBUG [RS:1;61c2066a8a0f:42993 {}] bucket.BucketCache(1509): Thread RS:1;61c2066a8a0f:42993 finished persisting bucket cache to file, renaming 2024-11-20T06:55:02,076 DEBUG [RS:0;61c2066a8a0f:45557 {}] bucket.BucketCache(1509): Thread RS:0;61c2066a8a0f:45557 finished persisting bucket cache to file, renaming 2024-11-20T06:55:02,076 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T06:55:02,076 INFO [RS:1;61c2066a8a0f:42993 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T06:55:02,076 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T06:55:02,076 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T06:55:02,076 INFO [RS:1;61c2066a8a0f:42993 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T06:55:02,076 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T06:55:02,076 INFO [RS:0;61c2066a8a0f:45557 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T06:55:02,076 INFO [RS:0;61c2066a8a0f:45557 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T06:55:02,076 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(3091): Received CLOSE for 6965edae25549910ad73492f24200332 2024-11-20T06:55:02,077 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,45557,1732085697147 2024-11-20T06:55:02,077 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:55:02,077 INFO [RS:0;61c2066a8a0f:45557 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:45557. 2024-11-20T06:55:02,077 DEBUG [RS:0;61c2066a8a0f:45557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:02,077 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,42993,1732085697306 2024-11-20T06:55:02,077 DEBUG [RS:0;61c2066a8a0f:45557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:02,077 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:55:02,077 INFO [RS:1;61c2066a8a0f:42993 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;61c2066a8a0f:42993. 2024-11-20T06:55:02,077 DEBUG [RS:1;61c2066a8a0f:42993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:02,077 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,45557,1732085697147; all regions closed. 2024-11-20T06:55:02,077 DEBUG [RS:1;61c2066a8a0f:42993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:02,078 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T06:55:02,078 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T06:55:02,078 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T06:55:02,078 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T06:55:02,078 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T06:55:02,078 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6965edae25549910ad73492f24200332, disabling compactions & flushes 2024-11-20T06:55:02,078 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:02,078 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:02,078 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. after waiting 0 ms 2024-11-20T06:55:02,078 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:02,079 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T06:55:02,079 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6965edae25549910ad73492f24200332=testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332.} 2024-11-20T06:55:02,079 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T06:55:02,079 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T06:55:02,079 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T06:55:02,079 DEBUG [StoreFileCloser-6965edae25549910ad73492f24200332-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/family/d322894e9a0d4a599d2a01270075492c 2024-11-20T06:55:02,079 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T06:55:02,079 DEBUG [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6965edae25549910ad73492f24200332 2024-11-20T06:55:02,079 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T06:55:02,079 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T06:55:02,079 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-20T06:55:02,083 DEBUG [RS:0;61c2066a8a0f:45557 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs 2024-11-20T06:55:02,083 INFO [RS:0;61c2066a8a0f:45557 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C45557%2C1732085697147:(num 1732085698817) 2024-11-20T06:55:02,083 DEBUG [RS:0;61c2066a8a0f:45557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:02,083 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:02,083 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:55:02,084 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T06:55:02,084 INFO [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:55:02,084 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T06:55:02,084 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T06:55:02,084 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T06:55:02,084 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:55:02,084 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnRegionMove1732085699642/6965edae25549910ad73492f24200332/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-20T06:55:02,085 INFO [RS:0;61c2066a8a0f:45557 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45557 2024-11-20T06:55:02,085 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:02,085 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6965edae25549910ad73492f24200332: Waiting for close lock at 1732085702077Running coprocessor pre-close hooks at 1732085702078 (+1 ms)Disabling compacts and flushes for region at 1732085702078Disabling writes for close at 1732085702078Writing region close event to WAL at 1732085702079 (+1 ms)Running coprocessor post-close hooks at 1732085702085 (+6 ms)Closed at 1732085702085 2024-11-20T06:55:02,085 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332. 2024-11-20T06:55:02,090 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 33797f5ee1be459ba6be11cd10f5f1de 2024-11-20T06:55:02,105 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/33797f5ee1be459ba6be11cd10f5f1de is 205, key is testBlockEvictionOnRegionMove1732085699642,,1732085699662.6965edae25549910ad73492f24200332./info:regioninfo/1732085702003/Put/seqid=0 2024-11-20T06:55:02,106 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/33797f5ee1be459ba6be11cd10f5f1de 2024-11-20T06:55:02,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:55:02,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,45557,1732085697147 2024-11-20T06:55:02,109 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:55:02,110 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,45557,1732085697147] 2024-11-20T06:55:02,115 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 43656366836149cf8b6901d193bc9d59 2024-11-20T06:55:02,129 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,45557,1732085697147 already deleted, retry=false 2024-11-20T06:55:02,129 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,45557,1732085697147 expired; onlineServers=1 2024-11-20T06:55:02,134 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/ns/43656366836149cf8b6901d193bc9d59 is 43, key is default/ns:d/1732085699470/Put/seqid=0 2024-11-20T06:55:02,135 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/ns/43656366836149cf8b6901d193bc9d59 2024-11-20T06:55:02,145 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e866ba4a383f43e681f6ddb6dd3edf4f 2024-11-20T06:55:02,158 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/table/e866ba4a383f43e681f6ddb6dd3edf4f is 78, key is testBlockEvictionOnRegionMove1732085699642/table:state/1732085700074/Put/seqid=0 2024-11-20T06:55:02,159 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/table/e866ba4a383f43e681f6ddb6dd3edf4f 2024-11-20T06:55:02,161 INFO [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:02,161 INFO [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:02,161 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/33797f5ee1be459ba6be11cd10f5f1de as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/33797f5ee1be459ba6be11cd10f5f1de 2024-11-20T06:55:02,162 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/33797f5ee1be459ba6be11cd10f5f1de, delay=948 ms 2024-11-20T06:55:02,164 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/33797f5ee1be459ba6be11cd10f5f1de, entries=15, sequenceid=15, filesize=7.6 K 2024-11-20T06:55:02,164 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/ns/43656366836149cf8b6901d193bc9d59 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/43656366836149cf8b6901d193bc9d59 2024-11-20T06:55:02,165 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/43656366836149cf8b6901d193bc9d59, delay=911 ms 2024-11-20T06:55:02,167 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/43656366836149cf8b6901d193bc9d59, entries=2, sequenceid=15, filesize=5.0 K 2024-11-20T06:55:02,167 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/table/e866ba4a383f43e681f6ddb6dd3edf4f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/e866ba4a383f43e681f6ddb6dd3edf4f 2024-11-20T06:55:02,169 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/e866ba4a383f43e681f6ddb6dd3edf4f, delay=978 ms 2024-11-20T06:55:02,171 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/e866ba4a383f43e681f6ddb6dd3edf4f, entries=2, sequenceid=15, filesize=5.3 K 2024-11-20T06:55:02,172 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-11-20T06:55:02,172 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T06:55:02,174 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/33797f5ee1be459ba6be11cd10f5f1de 2024-11-20T06:55:02,175 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/43656366836149cf8b6901d193bc9d59 2024-11-20T06:55:02,176 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/e866ba4a383f43e681f6ddb6dd3edf4f 2024-11-20T06:55:02,180 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-20T06:55:02,181 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T06:55:02,181 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T06:55:02,182 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732085702079Running coprocessor pre-close hooks at 1732085702079Disabling compacts and flushes for region at 1732085702079Disabling writes for close at 1732085702079Obtaining lock to block concurrent updates at 1732085702080 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732085702080Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732085702080Flushing stores of hbase:meta,,1.1588230740 at 1732085702080Flushing 1588230740/info: creating writer at 1732085702081 (+1 ms)Flushing 1588230740/info: appending metadata at 1732085702104 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732085702104Flushing 1588230740/ns: creating writer at 1732085702108 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732085702133 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1732085702133Flushing 1588230740/table: creating writer at 1732085702139 (+6 ms)Flushing 1588230740/table: appending metadata at 1732085702157 (+18 ms)Flushing 1588230740/table: closing flushed file at 1732085702157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@545396c0: reopening flushed file at 1732085702161 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6350402b: reopening flushed file at 1732085702164 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46b19710: reopening flushed file at 1732085702167 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1732085702172 (+5 ms)Writing region close event to WAL at 1732085702176 (+4 ms)Running coprocessor post-close hooks at 1732085702181 (+5 ms)Closed at 1732085702181 2024-11-20T06:55:02,182 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T06:55:02,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:02,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45557-0x101584fd8990001, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:02,221 INFO [RS:0;61c2066a8a0f:45557 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:55:02,222 INFO [RS:0;61c2066a8a0f:45557 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,45557,1732085697147; zookeeper connection closed. 2024-11-20T06:55:02,222 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6944a8d3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6944a8d3 2024-11-20T06:55:02,279 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,42993,1732085697306; all regions closed. 2024-11-20T06:55:02,286 DEBUG [RS:1;61c2066a8a0f:42993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs 2024-11-20T06:55:02,286 INFO [RS:1;61c2066a8a0f:42993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C42993%2C1732085697306.meta:.meta(num 1732085699333) 2024-11-20T06:55:02,288 DEBUG [RS:1;61c2066a8a0f:42993 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs 2024-11-20T06:55:02,289 INFO [RS:1;61c2066a8a0f:42993 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C42993%2C1732085697306:(num 1732085698817) 2024-11-20T06:55:02,289 DEBUG [RS:1;61c2066a8a0f:42993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:02,289 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:02,289 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:55:02,289 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T06:55:02,289 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:55:02,289 INFO [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:55:02,289 INFO [RS:1;61c2066a8a0f:42993 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42993 2024-11-20T06:55:02,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:55:02,315 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,42993,1732085697306 2024-11-20T06:55:02,315 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:55:02,326 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,42993,1732085697306] 2024-11-20T06:55:02,337 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,42993,1732085697306 already deleted, retry=false 2024-11-20T06:55:02,337 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,42993,1732085697306 expired; onlineServers=0 2024-11-20T06:55:02,337 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,34697,1732085696275' ***** 2024-11-20T06:55:02,337 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T06:55:02,337 INFO [M:0;61c2066a8a0f:34697 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:55:02,337 INFO [M:0;61c2066a8a0f:34697 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:55:02,337 DEBUG [M:0;61c2066a8a0f:34697 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T06:55:02,338 DEBUG [M:0;61c2066a8a0f:34697 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T06:55:02,338 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T06:55:02,338 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085698492 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085698492,5,FailOnTimeoutGroup] 2024-11-20T06:55:02,338 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085698499 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085698499,5,FailOnTimeoutGroup] 2024-11-20T06:55:02,338 INFO [M:0;61c2066a8a0f:34697 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T06:55:02,338 INFO [M:0;61c2066a8a0f:34697 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:55:02,338 DEBUG [M:0;61c2066a8a0f:34697 {}] master.HMaster(1795): Stopping service threads 2024-11-20T06:55:02,338 INFO [M:0;61c2066a8a0f:34697 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T06:55:02,338 INFO [M:0;61c2066a8a0f:34697 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T06:55:02,339 INFO [M:0;61c2066a8a0f:34697 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T06:55:02,339 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T06:55:02,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T06:55:02,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,346 DEBUG [M:0;61c2066a8a0f:34697 {}] zookeeper.ZKUtil(347): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T06:55:02,346 WARN [M:0;61c2066a8a0f:34697 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T06:55:02,346 INFO [M:0;61c2066a8a0f:34697 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.lastflushedseqids 2024-11-20T06:55:02,356 INFO [M:0;61c2066a8a0f:34697 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T06:55:02,356 INFO [M:0;61c2066a8a0f:34697 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T06:55:02,357 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T06:55:02,357 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:02,357 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:02,357 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T06:55:02,357 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:02,357 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-20T06:55:02,366 DEBUG [M:0;61c2066a8a0f:34697 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1442e4764eae4dbcb31ce70f7b2279c3 2024-11-20T06:55:02,379 DEBUG [M:0;61c2066a8a0f:34697 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1442e4764eae4dbcb31ce70f7b2279c3 is 82, key is hbase:meta,,1/info:regioninfo/1732085699395/Put/seqid=0 2024-11-20T06:55:02,380 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1442e4764eae4dbcb31ce70f7b2279c3 2024-11-20T06:55:02,387 DEBUG [M:0;61c2066a8a0f:34697 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aa204e875bbc4817815bab398141b081 2024-11-20T06:55:02,400 DEBUG [M:0;61c2066a8a0f:34697 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa204e875bbc4817815bab398141b081 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732085700080/Put/seqid=0 2024-11-20T06:55:02,401 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa204e875bbc4817815bab398141b081 2024-11-20T06:55:02,408 DEBUG [M:0;61c2066a8a0f:34697 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dccd9036a8134efc8e430f7a32bc0900 2024-11-20T06:55:02,420 DEBUG [M:0;61c2066a8a0f:34697 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dccd9036a8134efc8e430f7a32bc0900 is 69, key is 61c2066a8a0f,42993,1732085697306/rs:state/1732085698524/Put/seqid=0 2024-11-20T06:55:02,420 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dccd9036a8134efc8e430f7a32bc0900 2024-11-20T06:55:02,422 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1442e4764eae4dbcb31ce70f7b2279c3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1442e4764eae4dbcb31ce70f7b2279c3 2024-11-20T06:55:02,423 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1442e4764eae4dbcb31ce70f7b2279c3, entries=8, sequenceid=84, filesize=5.5 K 2024-11-20T06:55:02,424 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/aa204e875bbc4817815bab398141b081 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa204e875bbc4817815bab398141b081 2024-11-20T06:55:02,425 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/aa204e875bbc4817815bab398141b081, entries=9, sequenceid=84, filesize=6.5 K 2024-11-20T06:55:02,425 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/dccd9036a8134efc8e430f7a32bc0900 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dccd9036a8134efc8e430f7a32bc0900 2024-11-20T06:55:02,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:02,426 INFO [RS:1;61c2066a8a0f:42993 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:55:02,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42993-0x101584fd8990002, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:02,426 INFO [RS:1;61c2066a8a0f:42993 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,42993,1732085697306; zookeeper connection closed. 2024-11-20T06:55:02,426 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1044f319 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1044f319 2024-11-20T06:55:02,427 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-20T06:55:02,427 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/dccd9036a8134efc8e430f7a32bc0900, entries=2, sequenceid=84, filesize=5.1 K 2024-11-20T06:55:02,428 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=84, compaction requested=false 2024-11-20T06:55:02,430 INFO [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:02,430 DEBUG [M:0;61c2066a8a0f:34697 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732085702356Disabling compacts and flushes for region at 1732085702356Disabling writes for close at 1732085702357 (+1 ms)Obtaining lock to block concurrent updates at 1732085702357Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085702357Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732085702358 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085702358Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085702358Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085702379 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085702379Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085702382 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085702400 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085702400Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085702403 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085702419 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085702419Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f4218e3: reopening flushed file at 1732085702422 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b8713e3: reopening flushed file at 1732085702423 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@43c52c5a: reopening flushed file at 1732085702425 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=84, compaction requested=false at 1732085702428 (+3 ms)Writing region close event to WAL at 1732085702429 (+1 ms)Closed at 1732085702430 (+1 ms) 2024-11-20T06:55:02,431 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:55:02,431 INFO [M:0;61c2066a8a0f:34697 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T06:55:02,432 INFO [M:0;61c2066a8a0f:34697 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34697 2024-11-20T06:55:02,432 INFO [M:0;61c2066a8a0f:34697 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:55:02,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:02,562 INFO [M:0;61c2066a8a0f:34697 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:55:02,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34697-0x101584fd8990000, quorum=127.0.0.1:54382, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:02,571 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T06:55:02,595 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T06:55:02,595 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,605 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/61c2066a8a0f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/61c2066a8a0f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732085700444 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732085702165 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/61c2066a8a0f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/61c2066a8a0f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732085702162 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732085701990 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=409 (was 393) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4703 (was 5107) 2024-11-20T06:55:02,611 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=409, ProcessCount=11, AvailableMemoryMB=4703 2024-11-20T06:55:02,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/hadoop-log-dir so I do NOT create it in target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be 2024-11-20T06:55:02,612 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/hadoop-tmp-dir so I do NOT create it in target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be 2024-11-20T06:55:02,612 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,615 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/cluster_8b1c93a0-2938-3e01-781a-3b01ef3bad3c, deleteOnExit=true 2024-11-20T06:55:02,618 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/cluster_8b1c93a0-2938-3e01-781a-3b01ef3bad3c/zookeeper_0, clientPort=61976, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/cluster_8b1c93a0-2938-3e01-781a-3b01ef3bad3c/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/cluster_8b1c93a0-2938-3e01-781a-3b01ef3bad3c/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T06:55:02,619 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61976 2024-11-20T06:55:02,628 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 with version=8 2024-11-20T06:55:02,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/hbase-staging 2024-11-20T06:55:02,631 INFO [Time-limited test {}] client.ConnectionUtils(128): master/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:55:02,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,632 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:55:02,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:55:02,632 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T06:55:02,632 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:55:02,633 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40491 2024-11-20T06:55:02,635 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40491 connecting to ZooKeeper ensemble=127.0.0.1:61976 2024-11-20T06:55:02,696 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404910x0, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:55:02,697 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40491-0x101584ff29e0000 connected 2024-11-20T06:55:02,779 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,780 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,781 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:02,782 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63, hbase.cluster.distributed=false 2024-11-20T06:55:02,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:55:02,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40491 2024-11-20T06:55:02,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40491 2024-11-20T06:55:02,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40491 2024-11-20T06:55:02,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40491 2024-11-20T06:55:02,788 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40491 2024-11-20T06:55:02,808 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:55:02,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,808 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:55:02,808 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,809 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:55:02,809 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T06:55:02,809 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:55:02,810 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43717 2024-11-20T06:55:02,811 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43717 connecting to ZooKeeper ensemble=127.0.0.1:61976 2024-11-20T06:55:02,811 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,812 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:437170x0, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:55:02,821 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43717-0x101584ff29e0001 connected 2024-11-20T06:55:02,821 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:02,822 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T06:55:02,822 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.cache 2024-11-20T06:55:02,823 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T06:55:02,823 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T06:55:02,823 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T06:55:02,823 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T06:55:02,823 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T06:55:02,823 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T06:55:02,824 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T06:55:02,824 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T06:55:02,824 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence is complete. 2024-11-20T06:55:02,824 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T06:55:02,826 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:55:02,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43717 2024-11-20T06:55:02,827 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43717 2024-11-20T06:55:02,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43717 2024-11-20T06:55:02,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43717 2024-11-20T06:55:02,828 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43717 2024-11-20T06:55:02,851 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:55:02,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,852 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:55:02,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:02,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:55:02,852 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T06:55:02,852 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:55:02,853 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36177 2024-11-20T06:55:02,855 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36177 connecting to ZooKeeper ensemble=127.0.0.1:61976 2024-11-20T06:55:02,856 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,856 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361770x0, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:55:02,869 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36177-0x101584ff29e0002 connected 2024-11-20T06:55:02,869 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:02,869 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T06:55:02,870 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.cache 2024-11-20T06:55:02,870 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T06:55:02,870 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T06:55:02,870 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T06:55:02,871 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T06:55:02,871 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T06:55:02,871 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-20T06:55:02,871 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T06:55:02,871 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence is complete. 2024-11-20T06:55:02,871 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T06:55:02,872 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T06:55:02,873 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:55:02,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36177 2024-11-20T06:55:02,874 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36177 2024-11-20T06:55:02,878 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36177 2024-11-20T06:55:02,879 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36177 2024-11-20T06:55:02,881 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36177 2024-11-20T06:55:02,895 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;61c2066a8a0f:40491 2024-11-20T06:55:02,895 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:02,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:55:02,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:55:02,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:55:02,905 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:02,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T06:55:02,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T06:55:02,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,914 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T06:55:02,914 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/61c2066a8a0f,40491,1732085702631 from backup master directory 2024-11-20T06:55:02,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:02,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:55:02,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:55:02,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T06:55:02,921 WARN [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:55:02,921 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:02,923 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/hbase.id] with ID: b587ccda-6259-41bc-a0ed-667ddd66d244 2024-11-20T06:55:02,923 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.tmp/hbase.id 2024-11-20T06:55:02,927 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/hbase.id] 2024-11-20T06:55:02,967 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:02,967 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T06:55:02,971 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 4ms. 2024-11-20T06:55:02,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:02,993 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,001 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:55:03,003 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T06:55:03,003 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:55:03,021 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store 2024-11-20T06:55:03,027 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:03,027 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T06:55:03,027 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:03,027 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:03,027 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T06:55:03,027 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:03,027 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:03,028 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732085703027Disabling compacts and flushes for region at 1732085703027Disabling writes for close at 1732085703027Writing region close event to WAL at 1732085703027Closed at 1732085703027 2024-11-20T06:55:03,028 WARN [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/.initializing 2024-11-20T06:55:03,028 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/WALs/61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:03,034 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:55:03,038 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=61c2066a8a0f%2C40491%2C1732085702631, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/WALs/61c2066a8a0f,40491,1732085702631, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/oldWALs, maxLogs=10 2024-11-20T06:55:03,056 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/WALs/61c2066a8a0f,40491,1732085702631/61c2066a8a0f%2C40491%2C1732085702631.1732085703038 2024-11-20T06:55:03,057 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:55:03,057 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:03,057 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:03,057 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,057 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,062 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T06:55:03,063 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:03,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T06:55:03,067 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,067 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:03,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T06:55:03,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:03,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,074 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T06:55:03,075 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:03,075 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,076 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,076 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,076 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,076 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,076 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:55:03,077 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T06:55:03,082 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:55:03,082 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65364392, jitterRate=-0.02599465847015381}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:55:03,083 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732085703057Initializing all the Stores at 1732085703058 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085703058Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085703058Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085703058Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085703058Cleaning up temporary data from old regions at 1732085703076 (+18 ms)Region opened successfully at 1732085703083 (+7 ms) 2024-11-20T06:55:03,083 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T06:55:03,087 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b0ad34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:55:03,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T06:55:03,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T06:55:03,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T06:55:03,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T06:55:03,089 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T06:55:03,089 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-20T06:55:03,089 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T06:55:03,091 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T06:55:03,092 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T06:55:03,126 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T06:55:03,127 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T06:55:03,128 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T06:55:03,137 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T06:55:03,138 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T06:55:03,140 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T06:55:03,146 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T06:55:03,147 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T06:55:03,154 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T06:55:03,157 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T06:55:03,168 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T06:55:03,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:03,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:03,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:03,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,180 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=61c2066a8a0f,40491,1732085702631, sessionid=0x101584ff29e0000, setting cluster-up flag (Was=false) 2024-11-20T06:55:03,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,221 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T06:55:03,222 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:03,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,262 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T06:55:03,264 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:03,264 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T06:55:03,266 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T06:55:03,266 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T06:55:03,266 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T06:55:03,267 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 61c2066a8a0f,40491,1732085702631 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T06:55:03,269 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:55:03,269 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:55:03,270 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:55:03,270 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=5, maxPoolSize=5 2024-11-20T06:55:03,270 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/61c2066a8a0f:0, corePoolSize=10, maxPoolSize=10 2024-11-20T06:55:03,270 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,270 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:55:03,270 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,271 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732085733271 2024-11-20T06:55:03,271 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T06:55:03,271 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T06:55:03,271 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T06:55:03,271 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T06:55:03,271 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T06:55:03,271 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T06:55:03,272 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,272 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:55:03,272 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T06:55:03,272 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T06:55:03,272 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T06:55:03,272 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T06:55:03,272 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T06:55:03,273 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,273 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T06:55:03,273 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085703273,5,FailOnTimeoutGroup] 2024-11-20T06:55:03,273 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T06:55:03,274 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085703273,5,FailOnTimeoutGroup] 2024-11-20T06:55:03,274 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,275 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T06:55:03,275 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,275 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,284 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(746): ClusterId : b587ccda-6259-41bc-a0ed-667ddd66d244 2024-11-20T06:55:03,284 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(746): ClusterId : b587ccda-6259-41bc-a0ed-667ddd66d244 2024-11-20T06:55:03,284 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T06:55:03,284 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T06:55:03,294 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T06:55:03,294 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T06:55:03,294 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T06:55:03,294 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T06:55:03,294 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T06:55:03,294 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:55:03,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:03,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T06:55:03,304 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T06:55:03,304 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:03,304 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T06:55:03,305 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T06:55:03,305 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T06:55:03,305 DEBUG [RS:1;61c2066a8a0f:36177 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@260f2f90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:55:03,305 DEBUG [RS:0;61c2066a8a0f:43717 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c05828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:55:03,307 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T06:55:03,307 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:03,308 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T06:55:03,312 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T06:55:03,312 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:03,312 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T06:55:03,319 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;61c2066a8a0f:36177 2024-11-20T06:55:03,319 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;61c2066a8a0f:43717 2024-11-20T06:55:03,319 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T06:55:03,319 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T06:55:03,319 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T06:55:03,319 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T06:55:03,319 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T06:55:03,319 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T06:55:03,320 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T06:55:03,320 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:03,320 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,40491,1732085702631 with port=36177, startcode=1732085702851 2024-11-20T06:55:03,320 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,40491,1732085702631 with port=43717, startcode=1732085702808 2024-11-20T06:55:03,320 DEBUG [RS:1;61c2066a8a0f:36177 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T06:55:03,320 DEBUG [RS:0;61c2066a8a0f:43717 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T06:55:03,320 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:03,320 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T06:55:03,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:55:03,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:55:03,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T06:55:03,321 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T06:55:03,321 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:55:03,322 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T06:55:03,327 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35385, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T06:55:03,327 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45637, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T06:55:03,328 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40491 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:03,328 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40491 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:03,330 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:55:03,330 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40491 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:03,330 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40491 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:03,330 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65308838, jitterRate=-0.02682247757911682}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:55:03,330 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:55:03,330 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T06:55:03,330 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T06:55:03,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732085703300Initializing all the Stores at 1732085703300Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085703300Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085703301 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085703301Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085703301Cleaning up temporary data from old regions at 1732085703321 (+20 ms)Region opened successfully at 1732085703331 (+10 ms) 2024-11-20T06:55:03,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T06:55:03,331 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T06:55:03,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T06:55:03,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T06:55:03,331 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T06:55:03,331 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:55:03,331 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T06:55:03,331 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T06:55:03,331 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T06:55:03,332 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732085703331Disabling compacts and flushes for region at 1732085703331Disabling writes for close at 1732085703331Writing region close event to WAL at 1732085703331Closed at 1732085703331 2024-11-20T06:55:03,332 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:55:03,332 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T06:55:03,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T06:55:03,333 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:55:03,334 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T06:55:03,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:55:03,380 DEBUG [RS:0;61c2066a8a0f:43717 {}] zookeeper.ZKUtil(111): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:03,380 WARN [RS:0;61c2066a8a0f:43717 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:55:03,380 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,43717,1732085702808] 2024-11-20T06:55:03,380 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,36177,1732085702851] 2024-11-20T06:55:03,380 INFO [RS:0;61c2066a8a0f:43717 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:55:03,380 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:03,380 DEBUG [RS:1;61c2066a8a0f:36177 {}] zookeeper.ZKUtil(111): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:03,381 WARN [RS:1;61c2066a8a0f:36177 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:55:03,381 INFO [RS:1;61c2066a8a0f:36177 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:55:03,381 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:03,385 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T06:55:03,385 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T06:55:03,387 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T06:55:03,387 INFO [RS:0;61c2066a8a0f:43717 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T06:55:03,388 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,388 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T06:55:03,388 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T06:55:03,388 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,388 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 INFO [RS:1;61c2066a8a0f:36177 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:55:03,389 DEBUG [RS:0;61c2066a8a0f:43717 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:55:03,390 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T06:55:03,390 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T06:55:03,390 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,390 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,390 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,390 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,390 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:55:03,391 DEBUG [RS:1;61c2066a8a0f:36177 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:55:03,395 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43717,1732085702808-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,396 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,36177,1732085702851-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:55:03,411 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T06:55:03,411 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T06:55:03,411 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,43717,1732085702808-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,411 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,36177,1732085702851-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,412 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,412 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,412 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.Replication(171): 61c2066a8a0f,43717,1732085702808 started 2024-11-20T06:55:03,412 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.Replication(171): 61c2066a8a0f,36177,1732085702851 started 2024-11-20T06:55:03,427 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,427 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:03,427 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,36177,1732085702851, RpcServer on 61c2066a8a0f/172.17.0.2:36177, sessionid=0x101584ff29e0002 2024-11-20T06:55:03,427 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,43717,1732085702808, RpcServer on 61c2066a8a0f/172.17.0.2:43717, sessionid=0x101584ff29e0001 2024-11-20T06:55:03,427 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T06:55:03,427 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T06:55:03,427 DEBUG [RS:1;61c2066a8a0f:36177 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:03,427 DEBUG [RS:0;61c2066a8a0f:43717 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:03,427 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,36177,1732085702851' 2024-11-20T06:55:03,427 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,43717,1732085702808' 2024-11-20T06:55:03,427 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T06:55:03,427 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T06:55:03,428 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T06:55:03,428 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T06:55:03,428 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T06:55:03,428 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T06:55:03,429 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T06:55:03,429 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T06:55:03,429 DEBUG [RS:1;61c2066a8a0f:36177 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:03,429 DEBUG [RS:0;61c2066a8a0f:43717 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:03,429 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,36177,1732085702851' 2024-11-20T06:55:03,429 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,43717,1732085702808' 2024-11-20T06:55:03,429 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T06:55:03,429 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T06:55:03,429 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T06:55:03,429 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T06:55:03,429 DEBUG [RS:1;61c2066a8a0f:36177 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T06:55:03,429 DEBUG [RS:0;61c2066a8a0f:43717 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T06:55:03,430 INFO [RS:1;61c2066a8a0f:36177 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T06:55:03,430 INFO [RS:0;61c2066a8a0f:43717 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T06:55:03,430 INFO [RS:1;61c2066a8a0f:36177 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T06:55:03,430 INFO [RS:0;61c2066a8a0f:43717 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T06:55:03,484 WARN [61c2066a8a0f:40491 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-20T06:55:03,531 INFO [RS:0;61c2066a8a0f:43717 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:55:03,531 INFO [RS:1;61c2066a8a0f:36177 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:55:03,532 INFO [RS:1;61c2066a8a0f:36177 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C36177%2C1732085702851, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:55:03,532 INFO [RS:0;61c2066a8a0f:43717 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C43717%2C1732085702808, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,43717,1732085702808, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:55:03,555 INFO [RS:1;61c2066a8a0f:36177 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851/61c2066a8a0f%2C36177%2C1732085702851.1732085703537 2024-11-20T06:55:03,555 INFO [RS:0;61c2066a8a0f:43717 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,43717,1732085702808/61c2066a8a0f%2C43717%2C1732085702808.1732085703537 2024-11-20T06:55:03,555 DEBUG [RS:1;61c2066a8a0f:36177 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:55:03,555 DEBUG [RS:0;61c2066a8a0f:43717 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:55:03,735 DEBUG [61c2066a8a0f:40491 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T06:55:03,736 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(204): Hosts are {61c2066a8a0f=0} racks are {/default-rack=0} 2024-11-20T06:55:03,737 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T06:55:03,737 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T06:55:03,738 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T06:55:03,738 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T06:55:03,738 INFO [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T06:55:03,738 INFO [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T06:55:03,738 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T06:55:03,739 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:03,741 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,36177,1732085702851, state=OPENING 2024-11-20T06:55:03,796 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T06:55:03,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:03,815 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:55:03,815 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:03,815 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:03,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,36177,1732085702851}] 2024-11-20T06:55:03,816 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:03,972 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T06:55:03,976 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49361, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T06:55:03,979 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T06:55:03,979 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:55:03,979 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T06:55:03,980 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C36177%2C1732085702851.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:55:03,997 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851/61c2066a8a0f%2C36177%2C1732085702851.meta.1732085703981.meta 2024-11-20T06:55:03,997 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:55:03,997 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:03,998 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T06:55:03,998 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T06:55:03,998 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T06:55:03,998 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T06:55:03,998 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:03,998 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T06:55:03,998 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T06:55:03,999 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T06:55:04,000 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T06:55:04,000 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:04,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:04,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T06:55:04,001 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T06:55:04,001 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:04,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:04,002 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T06:55:04,002 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T06:55:04,002 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:04,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:04,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T06:55:04,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T06:55:04,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:04,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:04,004 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T06:55:04,004 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:55:04,005 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:55:04,005 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T06:55:04,005 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T06:55:04,005 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:55:04,006 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T06:55:04,006 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59760039, jitterRate=-0.1095060259103775}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:55:04,006 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T06:55:04,007 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732085703998Writing region info on filesystem at 1732085703998Initializing all the Stores at 1732085703999 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085703999Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085703999Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085703999Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085703999Cleaning up temporary data from old regions at 1732085704005 (+6 ms)Running coprocessor post-open hooks at 1732085704006 (+1 ms)Region opened successfully at 1732085704007 (+1 ms) 2024-11-20T06:55:04,008 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732085703971 2024-11-20T06:55:04,010 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T06:55:04,010 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T06:55:04,011 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:04,012 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,36177,1732085702851, state=OPEN 2024-11-20T06:55:04,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:04,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:04,055 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:04,055 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:04,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:04,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:04,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:04,059 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T06:55:04,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,36177,1732085702851 in 240 msec 2024-11-20T06:55:04,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T06:55:04,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 728 msec 2024-11-20T06:55:04,066 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T06:55:04,066 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T06:55:04,069 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:55:04,069 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,36177,1732085702851, seqNum=-1] 2024-11-20T06:55:04,069 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:55:04,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52883, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:55:04,076 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 810 msec 2024-11-20T06:55:04,076 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732085704076, completionTime=-1 2024-11-20T06:55:04,077 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-20T06:55:04,077 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-20T06:55:04,079 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-20T06:55:04,080 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732085764079 2024-11-20T06:55:04,080 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732085824080 2024-11-20T06:55:04,080 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-20T06:55:04,080 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T06:55:04,080 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40491,1732085702631-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:04,081 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40491,1732085702631-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:04,081 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40491,1732085702631-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:04,081 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-61c2066a8a0f:40491, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:04,081 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:04,081 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:04,085 DEBUG [master/61c2066a8a0f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T06:55:04,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.167sec 2024-11-20T06:55:04,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T06:55:04,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T06:55:04,088 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T06:55:04,089 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T06:55:04,089 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T06:55:04,089 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40491,1732085702631-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:55:04,089 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40491,1732085702631-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T06:55:04,090 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T06:55:04,090 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T06:55:04,090 INFO [master/61c2066a8a0f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,40491,1732085702631-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:04,185 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@143f7fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:55:04,185 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 61c2066a8a0f,40491,-1 for getting cluster id 2024-11-20T06:55:04,186 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T06:55:04,188 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b587ccda-6259-41bc-a0ed-667ddd66d244' 2024-11-20T06:55:04,188 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T06:55:04,188 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b587ccda-6259-41bc-a0ed-667ddd66d244" 2024-11-20T06:55:04,189 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f17cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:55:04,189 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [61c2066a8a0f,40491,-1] 2024-11-20T06:55:04,189 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T06:55:04,189 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:04,191 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57416, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T06:55:04,192 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2af0bbc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T06:55:04,193 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:55:04,194 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,36177,1732085702851, seqNum=-1] 2024-11-20T06:55:04,194 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:55:04,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:55:04,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:04,199 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T06:55:04,200 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:04,200 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@464232a5 2024-11-20T06:55:04,200 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T06:55:04,202 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T06:55:04,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40491 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732085704198', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T06:55:04,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40491 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 2024-11-20T06:55:04,205 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T06:55:04,205 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:04,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40491 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732085704198" procId is: 4 2024-11-20T06:55:04,206 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T06:55:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:55:04,216 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 93f3ae5252be7d8acdf57ee0e048ad45, NAME => 'testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732085704198', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:55:04,224 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:04,225 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(1722): Closing 93f3ae5252be7d8acdf57ee0e048ad45, disabling compactions & flushes 2024-11-20T06:55:04,225 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:04,225 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:04,225 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. after waiting 0 ms 2024-11-20T06:55:04,225 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:04,225 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:04,225 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732085704198-pool-0 {}] regionserver.HRegion(1676): Region close journal for 93f3ae5252be7d8acdf57ee0e048ad45: Waiting for close lock at 1732085704225Disabling compacts and flushes for region at 1732085704225Disabling writes for close at 1732085704225Writing region close event to WAL at 1732085704225Closed at 1732085704225 2024-11-20T06:55:04,226 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T06:55:04,227 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732085704226"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732085704226"}]},"ts":"1732085704226"} 2024-11-20T06:55:04,229 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T06:55:04,230 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T06:55:04,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732085704198","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085704230"}]},"ts":"1732085704230"} 2024-11-20T06:55:04,233 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732085704198, state=ENABLING in hbase:meta 2024-11-20T06:55:04,233 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {61c2066a8a0f=0} racks are {/default-rack=0} 2024-11-20T06:55:04,234 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T06:55:04,234 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T06:55:04,234 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T06:55:04,234 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T06:55:04,234 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T06:55:04,234 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T06:55:04,234 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T06:55:04,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN}] 2024-11-20T06:55:04,235 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN 2024-11-20T06:55:04,236 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN; state=OFFLINE, location=61c2066a8a0f,36177,1732085702851; forceNewPlan=false, retain=false 2024-11-20T06:55:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:55:04,387 INFO [61c2066a8a0f:40491 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T06:55:04,388 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=93f3ae5252be7d8acdf57ee0e048ad45, regionState=OPENING, regionLocation=61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:04,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN because future has completed 2024-11-20T06:55:04,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93f3ae5252be7d8acdf57ee0e048ad45, server=61c2066a8a0f,36177,1732085702851}] 2024-11-20T06:55:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:55:04,552 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:04,552 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 93f3ae5252be7d8acdf57ee0e048ad45, NAME => 'testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:04,553 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732085704198 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,554 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:04,554 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,554 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,556 INFO [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,561 INFO [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f3ae5252be7d8acdf57ee0e048ad45 columnFamilyName family 2024-11-20T06:55:04,561 DEBUG [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:04,561 INFO [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] regionserver.HStore(327): Store=93f3ae5252be7d8acdf57ee0e048ad45/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:04,561 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,562 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,562 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,563 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,563 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,564 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,571 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T06:55:04,571 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 93f3ae5252be7d8acdf57ee0e048ad45; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66350351, jitterRate=-0.011302724480628967}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T06:55:04,572 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:04,573 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 93f3ae5252be7d8acdf57ee0e048ad45: Running coprocessor pre-open hook at 1732085704554Writing region info on filesystem at 1732085704554Initializing all the Stores at 1732085704555 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085704555Cleaning up temporary data from old regions at 1732085704563 (+8 ms)Running coprocessor post-open hooks at 1732085704572 (+9 ms)Region opened successfully at 1732085704573 (+1 ms) 2024-11-20T06:55:04,574 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45., pid=6, masterSystemTime=1732085704546 2024-11-20T06:55:04,577 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:04,577 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:04,577 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=93f3ae5252be7d8acdf57ee0e048ad45, regionState=OPEN, openSeqNum=2, regionLocation=61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:04,581 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93f3ae5252be7d8acdf57ee0e048ad45, server=61c2066a8a0f,36177,1732085702851 because future has completed 2024-11-20T06:55:04,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T06:55:04,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 93f3ae5252be7d8acdf57ee0e048ad45, server=61c2066a8a0f,36177,1732085702851 in 189 msec 2024-11-20T06:55:04,587 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T06:55:04,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN in 351 msec 2024-11-20T06:55:04,588 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T06:55:04,589 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732085704198","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732085704588"}]},"ts":"1732085704588"} 2024-11-20T06:55:04,592 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732085704198, state=ENABLED in hbase:meta 2024-11-20T06:55:04,593 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T06:55:04,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732085704198 in 391 msec 2024-11-20T06:55:04,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40491 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T06:55:04,841 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732085704198 completed 2024-11-20T06:55:04,841 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732085704198 get assigned. Timeout = 60000ms 2024-11-20T06:55:04,841 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T06:55:04,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732085704198 assigned to meta. Checking AM states. 2024-11-20T06:55:04,847 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T06:55:04,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732085704198 assigned. 2024-11-20T06:55:04,854 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732085704198', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45., hostname=61c2066a8a0f,36177,1732085702851, seqNum=2] 2024-11-20T06:55:04,860 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 93f3ae5252be7d8acdf57ee0e048ad45 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-20T06:55:04,867 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 626763d261a74faca9ec8f6f9a9a92d8 2024-11-20T06:55:04,868 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T06:55:04,882 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/.tmp/family/626763d261a74faca9ec8f6f9a9a92d8 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-20T06:55:04,883 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/.tmp/family/626763d261a74faca9ec8f6f9a9a92d8 2024-11-20T06:55:04,887 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/.tmp/family/626763d261a74faca9ec8f6f9a9a92d8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8 2024-11-20T06:55:04,888 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, delay=996 ms 2024-11-20T06:55:04,890 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, entries=2, sequenceid=6, filesize=4.9 K 2024-11-20T06:55:04,891 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 93f3ae5252be7d8acdf57ee0e048ad45 in 31ms, sequenceid=6, compaction requested=false 2024-11-20T06:55:04,891 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732085704198' 2024-11-20T06:55:04,892 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 93f3ae5252be7d8acdf57ee0e048ad45: 2024-11-20T06:55:05,886 DEBUG [hfile-prefetch-1732085700444 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 626763d261a74faca9ec8f6f9a9a92d8_0. 2024-11-20T06:55:05,886 DEBUG [hfile-prefetch-1732085700444 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 626763d261a74faca9ec8f6f9a9a92d8_117. 2024-11-20T06:55:05,887 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, with total blocks 2, and data blocks 1 2024-11-20T06:55:05,887 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T06:55:05,887 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T06:55:05,887 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8. Will sleep 100ms and try the verification again. 2024-11-20T06:55:05,893 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;61c2066a8a0f:36177,5,FailOnTimeoutGroup] 2024-11-20T06:55:05,893 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,36177,1732085702851' ***** 2024-11-20T06:55:05,893 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-20T06:55:05,894 INFO [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T06:55:05,898 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T06:55:05,899 INFO [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:05,977 INFO [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence 2024-11-20T06:55:05,977 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T06:55:05,977 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T06:55:05,978 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T06:55:05,978 INFO [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T06:55:05,978 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T06:55:05,987 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, with total blocks 2, and data blocks 1 2024-11-20T06:55:05,988 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-20T06:55:05,988 DEBUG [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-20T06:55:05,988 INFO [hfile-prefetch-1732085700444 {}] bucket.BucketCache(2331): We found only 0 data blocks cached from a total of 1 for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, but no blocks pending caching. Maybe cache is full or evictions happened concurrently to cache prefetch. 2024-11-20T06:55:05,988 DEBUG [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1487): Thread RS:1;61c2066a8a0f:36177 started persisting bucket cache to file 2024-11-20T06:55:05,989 DEBUG [hfile-prefetch-1732085700444 {}] hfile.PrefetchExecutor(121): Prefetch completed for 626763d261a74faca9ec8f6f9a9a92d8 2024-11-20T06:55:05,989 DEBUG [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:05,989 DEBUG [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:05,996 DEBUG [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:05,996 DEBUG [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085705989 2024-11-20T06:55:05,996 DEBUG [RS:1;61c2066a8a0f:36177 {}] bucket.BucketCache(1509): Thread RS:1;61c2066a8a0f:36177 finished persisting bucket cache to file, renaming 2024-11-20T06:55:05,996 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T06:55:05,997 INFO [RS:1;61c2066a8a0f:36177 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T06:55:05,997 INFO [RS:1;61c2066a8a0f:36177 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T06:55:05,997 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(3091): Received CLOSE for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:05,997 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:05,997 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:55:05,998 INFO [RS:1;61c2066a8a0f:36177 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;61c2066a8a0f:36177. 2024-11-20T06:55:05,998 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T06:55:05,998 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 93f3ae5252be7d8acdf57ee0e048ad45, disabling compactions & flushes 2024-11-20T06:55:05,998 DEBUG [RS:1;61c2066a8a0f:36177 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:05,998 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:05,998 DEBUG [RS:1;61c2066a8a0f:36177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:05,998 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:05,998 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. after waiting 0 ms 2024-11-20T06:55:05,998 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T06:55:05,998 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:05,998 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T06:55:05,998 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T06:55:05,998 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T06:55:05,999 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-20T06:55:05,999 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1325): Online Regions={93f3ae5252be7d8acdf57ee0e048ad45=testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45., 1588230740=hbase:meta,,1.1588230740} 2024-11-20T06:55:05,999 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T06:55:05,999 DEBUG [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:05,999 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T06:55:05,999 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T06:55:05,999 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T06:55:06,000 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T06:55:06,000 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T06:55:06,000 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-20T06:55:06,005 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-20T06:55:06,005 INFO [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:06,006 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 93f3ae5252be7d8acdf57ee0e048ad45: Waiting for close lock at 1732085705997Running coprocessor pre-close hooks at 1732085705998 (+1 ms)Disabling compacts and flushes for region at 1732085705998Disabling writes for close at 1732085705998Writing region close event to WAL at 1732085705999 (+1 ms)Running coprocessor post-close hooks at 1732085706005 (+6 ms)Closed at 1732085706005 2024-11-20T06:55:06,006 DEBUG [RS_CLOSE_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:06,013 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 60a059f855dc428584106272ca251f2b 2024-11-20T06:55:06,026 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-20T06:55:06,026 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-20T06:55:06,027 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:06,027 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:06,029 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/60a059f855dc428584106272ca251f2b is 209, key is testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45./info:regioninfo/1732085704577/Put/seqid=0 2024-11-20T06:55:06,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:06,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085706026 2024-11-20T06:55:06,029 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-20T06:55:06,030 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/60a059f855dc428584106272ca251f2b 2024-11-20T06:55:06,038 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 487362b44dbf4032a7c91b7ab8aeba4e 2024-11-20T06:55:06,052 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/ns/487362b44dbf4032a7c91b7ab8aeba4e is 43, key is default/ns:d/1732085704072/Put/seqid=0 2024-11-20T06:55:06,052 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/ns/487362b44dbf4032a7c91b7ab8aeba4e 2024-11-20T06:55:06,060 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dfbcaa3549b248128bd17f9f780aef93 2024-11-20T06:55:06,072 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/table/dfbcaa3549b248128bd17f9f780aef93 is 80, key is testBlockEvictionOnGracefulStop1732085704198/table:state/1732085704588/Put/seqid=0 2024-11-20T06:55:06,072 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/table/dfbcaa3549b248128bd17f9f780aef93 2024-11-20T06:55:06,075 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/60a059f855dc428584106272ca251f2b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/60a059f855dc428584106272ca251f2b 2024-11-20T06:55:06,076 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/60a059f855dc428584106272ca251f2b, delay=960 ms 2024-11-20T06:55:06,077 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/60a059f855dc428584106272ca251f2b, entries=10, sequenceid=11, filesize=7.0 K 2024-11-20T06:55:06,077 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/ns/487362b44dbf4032a7c91b7ab8aeba4e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/487362b44dbf4032a7c91b7ab8aeba4e 2024-11-20T06:55:06,079 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/487362b44dbf4032a7c91b7ab8aeba4e, delay=953 ms 2024-11-20T06:55:06,079 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/487362b44dbf4032a7c91b7ab8aeba4e, entries=2, sequenceid=11, filesize=5.0 K 2024-11-20T06:55:06,080 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/table/dfbcaa3549b248128bd17f9f780aef93 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/dfbcaa3549b248128bd17f9f780aef93 2024-11-20T06:55:06,081 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/dfbcaa3549b248128bd17f9f780aef93, delay=914 ms 2024-11-20T06:55:06,082 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/dfbcaa3549b248128bd17f9f780aef93, entries=2, sequenceid=11, filesize=5.3 K 2024-11-20T06:55:06,083 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=11, compaction requested=false 2024-11-20T06:55:06,083 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/60a059f855dc428584106272ca251f2b 2024-11-20T06:55:06,084 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/487362b44dbf4032a7c91b7ab8aeba4e 2024-11-20T06:55:06,084 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/dfbcaa3549b248128bd17f9f780aef93 2024-11-20T06:55:06,088 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-20T06:55:06,089 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T06:55:06,089 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T06:55:06,089 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732085705999Running coprocessor pre-close hooks at 1732085705999Disabling compacts and flushes for region at 1732085705999Disabling writes for close at 1732085706000 (+1 ms)Obtaining lock to block concurrent updates at 1732085706000Preparing flush snapshotting stores in 1588230740 at 1732085706000Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732085706001 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732085706001Flushing 1588230740/info: creating writer at 1732085706001Flushing 1588230740/info: appending metadata at 1732085706028 (+27 ms)Flushing 1588230740/info: closing flushed file at 1732085706028Flushing 1588230740/ns: creating writer at 1732085706033 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732085706051 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732085706051Flushing 1588230740/table: creating writer at 1732085706054 (+3 ms)Flushing 1588230740/table: appending metadata at 1732085706071 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732085706071Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d562efb: reopening flushed file at 1732085706075 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@641ff7ce: reopening flushed file at 1732085706077 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77975c18: reopening flushed file at 1732085706080 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 82ms, sequenceid=11, compaction requested=false at 1732085706083 (+3 ms)Writing region close event to WAL at 1732085706085 (+2 ms)Running coprocessor post-close hooks at 1732085706089 (+4 ms)Closed at 1732085706089 2024-11-20T06:55:06,089 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T06:55:06,199 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,36177,1732085702851; all regions closed. 2024-11-20T06:55:06,204 DEBUG [RS:1;61c2066a8a0f:36177 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs 2024-11-20T06:55:06,204 INFO [RS:1;61c2066a8a0f:36177 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C36177%2C1732085702851.meta:.meta(num 1732085703981) 2024-11-20T06:55:06,209 DEBUG [RS:1;61c2066a8a0f:36177 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs 2024-11-20T06:55:06,209 INFO [RS:1;61c2066a8a0f:36177 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C36177%2C1732085702851:(num 1732085703537) 2024-11-20T06:55:06,209 DEBUG [RS:1;61c2066a8a0f:36177 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:06,209 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:06,210 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:55:06,210 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T06:55:06,211 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:55:06,211 INFO [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:55:06,211 INFO [RS:1;61c2066a8a0f:36177 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36177 2024-11-20T06:55:06,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:55:06,245 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,36177,1732085702851 2024-11-20T06:55:06,246 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:55:06,263 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,36177,1732085702851] 2024-11-20T06:55:06,279 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,36177,1732085702851 already deleted, retry=false 2024-11-20T06:55:06,279 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 61c2066a8a0f,36177,1732085702851 on 61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:06,283 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 61c2066a8a0f,36177,1732085702851, splitWal=true, meta=true 2024-11-20T06:55:06,286 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 61c2066a8a0f,36177,1732085702851 (carryingMeta=true) 61c2066a8a0f,36177,1732085702851/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@67774aa7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T06:55:06,286 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 61c2066a8a0f,36177,1732085702851, splitWal=true, meta=true 2024-11-20T06:55:06,287 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 61c2066a8a0f,36177,1732085702851, splitWal=true, meta=true, isMeta: true 2024-11-20T06:55:06,288 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851-splitting 2024-11-20T06:55:06,288 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851-splitting dir is empty, no logs to split. 2024-11-20T06:55:06,289 INFO [PEWorker-2 {}] master.SplitWALManager(105): 61c2066a8a0f,36177,1732085702851 WAL count=0, meta=true 2024-11-20T06:55:06,291 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851-splitting dir is empty, no logs to split. 2024-11-20T06:55:06,291 INFO [PEWorker-2 {}] master.SplitWALManager(105): 61c2066a8a0f,36177,1732085702851 WAL count=0, meta=true 2024-11-20T06:55:06,291 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 61c2066a8a0f,36177,1732085702851 WAL splitting is done? wals=0, meta=true 2024-11-20T06:55:06,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T06:55:06,294 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:55:06,294 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T06:55:06,363 INFO [RS:1;61c2066a8a0f:36177 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:55:06,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:06,363 INFO [RS:1;61c2066a8a0f:36177 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,36177,1732085702851; zookeeper connection closed. 2024-11-20T06:55:06,363 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36177-0x101584ff29e0002, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:06,363 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e83cb59 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e83cb59 2024-11-20T06:55:06,413 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/61c2066a8a0f:0 server-side Connection retries=45 2024-11-20T06:55:06,413 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:06,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:06,414 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T06:55:06,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T06:55:06,414 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T06:55:06,414 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T06:55:06,414 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T06:55:06,415 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46291 2024-11-20T06:55:06,416 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46291 connecting to ZooKeeper ensemble=127.0.0.1:61976 2024-11-20T06:55:06,416 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:06,416 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:06,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:462910x0, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T06:55:06,438 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46291-0x101584ff29e0003 connected 2024-11-20T06:55:06,438 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-20T06:55:06,438 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-20T06:55:06,439 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T06:55:06,440 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.cache 2024-11-20T06:55:06,440 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-20T06:55:06,441 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-20T06:55:06,441 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-20T06:55:06,442 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-20T06:55:06,442 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-20T06:55:06,443 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T06:55:06,444 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-20T06:55:06,444 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T06:55:06,444 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-20T06:55:06,445 DEBUG [61c2066a8a0f:40491 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T06:55:06,445 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:06,447 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,43717,1732085702808, state=OPENING 2024-11-20T06:55:06,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T06:55:06,454 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-20T06:55:06,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46291 2024-11-20T06:55:06,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46291 2024-11-20T06:55:06,455 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-20T06:55:06,455 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-20T06:55:06,455 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence is complete. 2024-11-20T06:55:06,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46291 2024-11-20T06:55:06,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46291 2024-11-20T06:55:06,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:06,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:06,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:06,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:06,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:06,462 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:06,462 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T06:55:06,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46291 2024-11-20T06:55:06,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,43717,1732085702808}] 2024-11-20T06:55:06,464 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(746): ClusterId : b587ccda-6259-41bc-a0ed-667ddd66d244 2024-11-20T06:55:06,464 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T06:55:06,480 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T06:55:06,480 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T06:55:06,496 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T06:55:06,497 DEBUG [RS:2;61c2066a8a0f:46291 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73bd00c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=61c2066a8a0f/172.17.0.2:0 2024-11-20T06:55:06,512 DEBUG [RS:2;61c2066a8a0f:46291 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;61c2066a8a0f:46291 2024-11-20T06:55:06,512 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T06:55:06,512 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T06:55:06,513 DEBUG [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T06:55:06,514 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(2659): reportForDuty to master=61c2066a8a0f,40491,1732085702631 with port=46291, startcode=1732085706413 2024-11-20T06:55:06,514 DEBUG [RS:2;61c2066a8a0f:46291 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T06:55:06,516 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47187, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T06:55:06,516 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40491 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:06,516 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40491 {}] master.ServerManager(517): Registering regionserver=61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:06,518 DEBUG [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63 2024-11-20T06:55:06,518 DEBUG [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-20T06:55:06,518 DEBUG [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T06:55:06,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:55:06,538 DEBUG [RS:2;61c2066a8a0f:46291 {}] zookeeper.ZKUtil(111): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:06,538 WARN [RS:2;61c2066a8a0f:46291 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T06:55:06,538 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [61c2066a8a0f,46291,1732085706413] 2024-11-20T06:55:06,538 INFO [RS:2;61c2066a8a0f:46291 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:55:06,538 DEBUG [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:06,543 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T06:55:06,546 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T06:55:06,547 INFO [RS:2;61c2066a8a0f:46291 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T06:55:06,547 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,547 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T06:55:06,548 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T06:55:06,548 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,548 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,548 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,548 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,548 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,548 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/61c2066a8a0f:0, corePoolSize=2, maxPoolSize=2 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/61c2066a8a0f:0, corePoolSize=1, maxPoolSize=1 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:55:06,549 DEBUG [RS:2;61c2066a8a0f:46291 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/61c2066a8a0f:0, corePoolSize=3, maxPoolSize=3 2024-11-20T06:55:06,550 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,550 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,550 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,550 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,550 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,550 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,46291,1732085706413-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T06:55:06,566 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T06:55:06,566 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=61c2066a8a0f,46291,1732085706413-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,567 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,567 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.Replication(171): 61c2066a8a0f,46291,1732085706413 started 2024-11-20T06:55:06,582 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T06:55:06,582 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(1482): Serving as 61c2066a8a0f,46291,1732085706413, RpcServer on 61c2066a8a0f/172.17.0.2:46291, sessionid=0x101584ff29e0003 2024-11-20T06:55:06,582 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T06:55:06,582 DEBUG [RS:2;61c2066a8a0f:46291 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:06,582 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,46291,1732085706413' 2024-11-20T06:55:06,582 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T06:55:06,583 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T06:55:06,584 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T06:55:06,584 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T06:55:06,584 DEBUG [RS:2;61c2066a8a0f:46291 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:06,584 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '61c2066a8a0f,46291,1732085706413' 2024-11-20T06:55:06,584 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T06:55:06,585 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T06:55:06,585 DEBUG [RS:2;61c2066a8a0f:46291 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T06:55:06,585 INFO [RS:2;61c2066a8a0f:46291 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T06:55:06,585 INFO [RS:2;61c2066a8a0f:46291 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T06:55:06,615 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T06:55:06,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54483, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T06:55:06,622 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T06:55:06,623 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T06:55:06,623 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T06:55:06,624 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C43717%2C1732085702808.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,43717,1732085702808, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:55:06,641 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,43717,1732085702808/61c2066a8a0f%2C43717%2C1732085702808.meta.1732085706624.meta 2024-11-20T06:55:06,641 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:55:06,641 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:06,641 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T06:55:06,642 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T06:55:06,642 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T06:55:06,642 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T06:55:06,642 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:06,642 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T06:55:06,642 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T06:55:06,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-20T06:55:06,643 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T06:55:06,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:06,645 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/60a059f855dc428584106272ca251f2b, delay=981 ms 2024-11-20T06:55:06,646 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/60a059f855dc428584106272ca251f2b 2024-11-20T06:55:06,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:06,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-20T06:55:06,647 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T06:55:06,648 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:06,650 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/487362b44dbf4032a7c91b7ab8aeba4e, delay=995 ms 2024-11-20T06:55:06,652 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/487362b44dbf4032a7c91b7ab8aeba4e 2024-11-20T06:55:06,652 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:06,653 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-20T06:55:06,653 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T06:55:06,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:06,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:06,654 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-20T06:55:06,655 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T06:55:06,655 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:06,657 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/dfbcaa3549b248128bd17f9f780aef93, delay=959 ms 2024-11-20T06:55:06,658 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/dfbcaa3549b248128bd17f9f780aef93 2024-11-20T06:55:06,658 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T06:55:06,659 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T06:55:06,659 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:55:06,660 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740 2024-11-20T06:55:06,660 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T06:55:06,660 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T06:55:06,660 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T06:55:06,661 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T06:55:06,662 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72899780, jitterRate=0.0862913727760315}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T06:55:06,662 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T06:55:06,663 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732085706642Writing region info on filesystem at 1732085706642Initializing all the Stores at 1732085706642Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085706642Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085706642Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085706642Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732085706642Cleaning up temporary data from old regions at 1732085706660 (+18 ms)Running coprocessor post-open hooks at 1732085706662 (+2 ms)Region opened successfully at 1732085706663 (+1 ms) 2024-11-20T06:55:06,664 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732085706615 2024-11-20T06:55:06,666 DEBUG [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T06:55:06,667 INFO [RS_OPEN_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T06:55:06,667 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:06,668 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 61c2066a8a0f,43717,1732085702808, state=OPEN 2024-11-20T06:55:06,686 INFO [RS:2;61c2066a8a0f:46291 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T06:55:06,687 INFO [RS:2;61c2066a8a0f:46291 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=61c2066a8a0f%2C46291%2C1732085706413, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,46291,1732085706413, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs, maxLogs=55 2024-11-20T06:55:06,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:06,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:06,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T06:55:06,704 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:06,704 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:06,704 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:06,704 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T06:55:06,706 INFO [RS:2;61c2066a8a0f:46291 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,46291,1732085706413/61c2066a8a0f%2C46291%2C1732085706413.1732085706687 2024-11-20T06:55:06,706 DEBUG [RS:2;61c2066a8a0f:46291 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-20T06:55:06,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-20T06:55:06,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=61c2066a8a0f,43717,1732085702808 in 242 msec 2024-11-20T06:55:06,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T06:55:06,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 414 msec 2024-11-20T06:55:06,710 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 61c2066a8a0f,36177,1732085702851 had 2 regions 2024-11-20T06:55:06,711 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 61c2066a8a0f,36177,1732085702851, splitWal=true, meta=true, isMeta: false 2024-11-20T06:55:06,712 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851-splitting dir is empty, no logs to split. 2024-11-20T06:55:06,712 INFO [PEWorker-3 {}] master.SplitWALManager(105): 61c2066a8a0f,36177,1732085702851 WAL count=0, meta=false 2024-11-20T06:55:06,713 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/WALs/61c2066a8a0f,36177,1732085702851-splitting dir is empty, no logs to split. 2024-11-20T06:55:06,713 INFO [PEWorker-3 {}] master.SplitWALManager(105): 61c2066a8a0f,36177,1732085702851 WAL count=0, meta=false 2024-11-20T06:55:06,713 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 61c2066a8a0f,36177,1732085702851 WAL splitting is done? wals=0, meta=false 2024-11-20T06:55:06,715 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN}] 2024-11-20T06:55:06,716 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN 2024-11-20T06:55:06,717 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T06:55:06,728 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T06:55:06,729 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T06:55:06,730 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T06:55:06,730 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T06:55:06,731 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T06:55:06,731 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T06:55:06,867 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(204): Hosts are {61c2066a8a0f=0} racks are {/default-rack=0} 2024-11-20T06:55:06,868 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T06:55:06,868 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T06:55:06,868 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T06:55:06,868 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T06:55:06,868 INFO [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T06:55:06,868 INFO [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T06:55:06,868 DEBUG [61c2066a8a0f:40491 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T06:55:06,869 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=93f3ae5252be7d8acdf57ee0e048ad45, regionState=OPENING, regionLocation=61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:06,870 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 61c2066a8a0f:36177 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 61c2066a8a0f/172.17.0.2:36177 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:55:06,871 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,36177,1732085702851, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,36177,1732085702851, seqNum=-1, error=java.net.ConnectException: Call to address=61c2066a8a0f:36177 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 61c2066a8a0f/172.17.0.2:36177 2024-11-20T06:55:06,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,36177,1732085702851, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T06:55:06,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,36177,1732085702851, seqNum=-1 from cache 2024-11-20T06:55:06,873 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 61c2066a8a0f:36177 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 61c2066a8a0f/172.17.0.2:36177 2024-11-20T06:55:06,978 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T06:55:06,979 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=61c2066a8a0f,43717,1732085702808, seqNum=-1] 2024-11-20T06:55:06,979 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T06:55:06,982 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44539, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T06:55:06,986 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732085704198, region=93f3ae5252be7d8acdf57ee0e048ad45, ASSIGN because future has completed 2024-11-20T06:55:06,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 93f3ae5252be7d8acdf57ee0e048ad45, server=61c2066a8a0f,46291,1732085706413}] 2024-11-20T06:55:07,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T06:55:07,083 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:55:07,083 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:07,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:07,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:07,083 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T06:55:07,083 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T06:55:07,083 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=796838939, stopped=false 2024-11-20T06:55:07,083 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=61c2066a8a0f,40491,1732085702631 2024-11-20T06:55:07,140 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T06:55:07,144 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-9-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56735, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T06:55:07,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:07,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:07,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:07,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T06:55:07,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:07,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:07,146 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T06:55:07,147 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T06:55:07,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:07,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:07,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:07,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:07,148 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T06:55:07,148 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,43717,1732085702808' ***** 2024-11-20T06:55:07,149 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T06:55:07,149 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. 2024-11-20T06:55:07,149 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '61c2066a8a0f,46291,1732085706413' ***** 2024-11-20T06:55:07,149 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T06:55:07,149 INFO [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T06:55:07,149 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => 93f3ae5252be7d8acdf57ee0e048ad45, NAME => 'testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45.', STARTKEY => '', ENDKEY => ''} 2024-11-20T06:55:07,149 INFO [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1738): Disabling cache 2024-11-20T06:55:07,150 INFO [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence 2024-11-20T06:55:07,150 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T06:55:07,150 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T06:55:07,150 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T06:55:07,150 INFO [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T06:55:07,150 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732085704198 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,150 INFO [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9169e6be-30f0-f8a1-53b1-e3f3daaa40be/bucket.persistence 2024-11-20T06:55:07,150 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T06:55:07,150 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-20T06:55:07,150 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T06:55:07,150 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-20T06:55:07,150 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-20T06:55:07,150 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T06:55:07,151 INFO [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-20T06:55:07,151 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,151 INFO [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:07,151 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,151 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-20T06:55:07,152 INFO [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,152 INFO [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93f3ae5252be7d8acdf57ee0e048ad45 columnFamilyName family 2024-11-20T06:55:07,152 DEBUG [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T06:55:07,154 DEBUG [StoreFileOpener-93f3ae5252be7d8acdf57ee0e048ad45-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8, delay=959 ms 2024-11-20T06:55:07,155 DEBUG [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45/family/626763d261a74faca9ec8f6f9a9a92d8 2024-11-20T06:55:07,155 INFO [StoreOpener-93f3ae5252be7d8acdf57ee0e048ad45-1 {}] regionserver.HStore(327): Store=93f3ae5252be7d8acdf57ee0e048ad45/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T06:55:07,155 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,156 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,157 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/default/testBlockEvictionOnGracefulStop1732085704198/93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,157 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,157 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,158 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,159 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened 93f3ae5252be7d8acdf57ee0e048ad45; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69248349, jitterRate=0.03188081085681915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T06:55:07,159 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 93f3ae5252be7d8acdf57ee0e048ad45 2024-11-20T06:55:07,160 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for 93f3ae5252be7d8acdf57ee0e048ad45: Running coprocessor pre-open hook at 1732085707151Writing region info on filesystem at 1732085707151Initializing all the Stores at 1732085707151Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732085707151Cleaning up temporary data from old regions at 1732085707157 (+6 ms)Running coprocessor post-open hooks at 1732085707159 (+2 ms)Region opened successfully at 1732085707160 (+1 ms) 2024-11-20T06:55:07,160 DEBUG [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1487): Thread RS:2;61c2066a8a0f:46291 started persisting bucket cache to file 2024-11-20T06:55:07,160 DEBUG [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:07,160 DEBUG [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:07,161 DEBUG [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1487): Thread RS:0;61c2066a8a0f:43717 started persisting bucket cache to file 2024-11-20T06:55:07,161 DEBUG [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-20T06:55:07,161 DEBUG [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-20T06:55:07,161 WARN [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(168): Fatal error occurred while opening region testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45., aborting... org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server 61c2066a8a0f,46291,1732085706413 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:55:07,162 ERROR [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2440): ***** ABORTING region server 61c2066a8a0f,46291,1732085706413: Failed to open region testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45. and can not recover ***** org.apache.hadoop.hbase.regionserver.RegionServerStoppedException: Server 61c2066a8a0f,46291,1732085706413 stopping at org.apache.hadoop.hbase.regionserver.RSRpcServices.checkOpen(RSRpcServices.java:1466) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2235) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T06:55:07,163 ERROR [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T06:55:07,163 DEBUG [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:07,163 DEBUG [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085707160 2024-11-20T06:55:07,163 DEBUG [RS:2;61c2066a8a0f:46291 {}] bucket.BucketCache(1509): Thread RS:2;61c2066a8a0f:46291 finished persisting bucket cache to file, renaming 2024-11-20T06:55:07,164 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T06:55:07,164 INFO [RS:2;61c2066a8a0f:46291 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T06:55:07,164 INFO [RS:2;61c2066a8a0f:46291 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T06:55:07,164 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(956): aborting server 61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:07,164 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:55:07,164 DEBUG [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-20T06:55:07,164 INFO [RS:2;61c2066a8a0f:46291 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;61c2066a8a0f:46291. 2024-11-20T06:55:07,164 DEBUG [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732085707161 2024-11-20T06:55:07,164 DEBUG [RS:0;61c2066a8a0f:43717 {}] bucket.BucketCache(1509): Thread RS:0;61c2066a8a0f:43717 finished persisting bucket cache to file, renaming 2024-11-20T06:55:07,164 DEBUG [RS:2;61c2066a8a0f:46291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:07,164 DEBUG [RS:2;61c2066a8a0f:46291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:07,164 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,46291,1732085706413; all regions closed. 2024-11-20T06:55:07,164 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T06:55:07,164 INFO [RS:0;61c2066a8a0f:43717 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T06:55:07,164 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(959): stopping server 61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;61c2066a8a0f:43717. 2024-11-20T06:55:07,165 DEBUG [RS:0;61c2066a8a0f:43717 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T06:55:07,165 DEBUG [RS:0;61c2066a8a0f:43717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T06:55:07,165 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T06:55:07,165 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T06:55:07,166 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T06:55:07,166 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T06:55:07,166 DEBUG [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T06:55:07,166 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-20T06:55:07,166 DEBUG [RS:2;61c2066a8a0f:46291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:07,166 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T06:55:07,166 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:07,166 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T06:55:07,166 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T06:55:07,166 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:55:07,166 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T06:55:07,166 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T06:55:07,166 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-11-20T06:55:07,166 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T06:55:07,167 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T06:55:07,167 INFO [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:55:07,167 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T06:55:07,167 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T06:55:07,167 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:55:07,167 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T06:55:07,167 INFO [RS:2;61c2066a8a0f:46291 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46291 2024-11-20T06:55:07,171 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T06:55:07,172 DEBUG [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T06:55:07,177 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d446debef498492ba7535f8968731995 2024-11-20T06:55:07,180 INFO [RS_OPEN_REGION-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 387650960 }, "NonHeapMemoryUsage": { "committed": 98762752, "init": 7667712, "max": -1, "used": 96481728 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "61c2066a8a0f", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11081, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 44, "ProcessCallTime_min": 0, "ProcessCallTime_max": 14, "ProcessCallTime_mean": 2, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 7, "ProcessCallTime_75th_percentile": 10, "ProcessCallTime_90th_percentile": 12, "ProcessCallTime_95th_percentile": 13, "ProcessCallTime_98th_percentile": 13, "ProcessCallTime_99th_percentile": 13, "ProcessCallTime_99.9th_percentile": 13, "ProcessCallTime_TimeRangeCount_0-1": 44, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 44, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 44, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 44, "TotalCallTime_min": 0, "TotalCallTime_max": 14, "TotalCallTime_mean": 2, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 7, "TotalCallTime_75th_percentile": 10, "TotalCallTime_90th_percentile": 12, "TotalCallTime_95th_percentile": 13, "TotalCallTime_98th_percentile": 13, "TotalCallTime_99th_percentile": 13, "TotalCallTime_99.9th_percentile": 13, "TotalCallTime_TimeRangeCount_0-1": 44, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 44, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 148, "ResponseSize_25th_percentile": 284, "ResponseSize_median": 569, "ResponseSize_75th_percentile": 853, "ResponseSize_90th_percentile": 1024, "ResponseSize_95th_percentile": 1081, "ResponseSize_98th_percentile": 1115, "ResponseSize_99th_percentile": 1126, "ResponseSize_99.9th_percentile": 1136, "ResponseSize_SizeRangeCount_0-10": 44, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 44, "RequestSize_min": 89, "RequestSize_max": 394, "RequestSize_mean": 174, "RequestSize_25th_percentile": 165, "RequestSize_median": 241, "RequestSize_75th_percentile": 317, "RequestSize_90th_percentile": 363, "RequestSize_95th_percentile": 378, "RequestSize_98th_percentile": 387, "RequestSize_99th_percentile": 390, "RequestSize_99.9th_percentile": 393, "RequestSize_SizeRangeCount_0-10": 44, "sentBytes": 5709 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "61c2066a8a0f", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:54382", "tag.serverName": "61c2066a8a0f,42993,1732085697306", "tag.clusterId": "a7a4a913-d216-4dbc-88aa-7f1007e89e7c", "tag.Context": "regionserver", "tag.Hostname": "61c2066a8a0f", "regionCount": 0, "storeCount": 0, "hlogFileCount": 2, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732085697306, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 0, "localBytesRead": 0, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 1340218376, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 418148352, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 19707, "ByteBuffAllocatorPoolAllocationBytes": 399360, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 19, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 1, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 7, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 11, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 7, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 1, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 1, "ScanTime_95th_percentile": 1, "ScanTime_98th_percentile": 1, "ScanTime_99th_percentile": 1, "ScanTime_99.9th_percentile": 1, "ScanTime_TimeRangeCount_0-1": 7, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 11, "Put_min": 1, "Put_max": 6, "Put_mean": 1, "Put_25th_percentile": 2, "Put_median": 3, "Put_75th_percentile": 4, "Put_90th_percentile": 5, "Put_95th_percentile": 5, "Put_98th_percentile": 5, "Put_99th_percentile": 5, "Put_99.9th_percentile": 5, "Put_TimeRangeCount_0-1": 11, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 18410, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 3789, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 18410, "FlushOutputSize_max": 18410, "FlushOutputSize_mean": 18410, "FlushOutputSize_25th_percentile": 18410, "FlushOutputSize_median": 18410, "FlushOutputSize_75th_percentile": 18410, "FlushOutputSize_90th_percentile": 18410, "FlushOutputSize_95th_percentile": 18410, "FlushOutputSize_98th_percentile": 18410, "FlushOutputSize_99th_percentile": 18410, "FlushOutputSize_99.9th_percentile": 18410, "FlushOutputSize_SizeRangeCount_0-10": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 3789, "FlushMemstoreSize_max": 3789, "FlushMemstoreSize_mean": 3789, "FlushMemstoreSize_25th_percentile": 3789, "FlushMemstoreSize_median": 3789, "FlushMemstoreSize_75th_percentile": 3789, "FlushMemstoreSize_90th_percentile": 3789, "FlushMemstoreSize_95th_percentile": 3789, "FlushMemstoreSize_98th_percentile": 3789, "FlushMemstoreSize_99th_percentile": 3789, "FlushMemstoreSize_99.9th_percentile": 3789, "FlushMemstoreSize_SizeRangeCount_0-10": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 1, "Get_min": 2, "Get_max": 2, "Get_mean": 2, "Get_25th_percentile": 2, "Get_median": 2, "Get_75th_percentile": 2, "Get_90th_percentile": 2, "Get_95th_percentile": 2, "Get_98th_percentile": 2, "Get_99th_percentile": 2, "Get_99.9th_percentile": 2, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 93, "FlushTime_max": 93, "FlushTime_mean": 93, "FlushTime_25th_percentile": 93, "FlushTime_median": 93, "FlushTime_75th_percentile": 93, "FlushTime_90th_percentile": 93, "FlushTime_95th_percentile": 93, "FlushTime_98th_percentile": 93, "FlushTime_99th_percentile": 93, "FlushTime_99.9th_percentile": 93, "FlushTime_TimeRangeCount_0-1": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 7, "ScanSize_min": 0, "ScanSize_max": 1112, "ScanSize_mean": 517, "ScanSize_25th_percentile": 278, "ScanSize_median": 556, "ScanSize_75th_percentile": 834, "ScanSize_90th_percentile": 1000, "ScanSize_95th_percentile": 1056, "ScanSize_98th_percentile": 1089, "ScanSize_99th_percentile": 1100, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_0-10": 7, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-20T06:55:07,191 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/d446debef498492ba7535f8968731995 is 209, key is testBlockEvictionOnGracefulStop1732085704198,,1732085704203.93f3ae5252be7d8acdf57ee0e048ad45./info:regioninfo/1732085706868/Put/seqid=0 2024-11-20T06:55:07,191 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/d446debef498492ba7535f8968731995 2024-11-20T06:55:07,194 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/.tmp/info/d446debef498492ba7535f8968731995 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/d446debef498492ba7535f8968731995 2024-11-20T06:55:07,195 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/d446debef498492ba7535f8968731995, delay=906 ms 2024-11-20T06:55:07,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,46291,1732085706413 2024-11-20T06:55:07,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:55:07,195 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:55:07,196 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/d446debef498492ba7535f8968731995, entries=3, sequenceid=18, filesize=5.9 K 2024-11-20T06:55:07,197 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=18, compaction requested=false 2024-11-20T06:55:07,197 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/60a059f855dc428584106272ca251f2b 2024-11-20T06:55:07,198 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/info/d446debef498492ba7535f8968731995 2024-11-20T06:55:07,198 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/ns/487362b44dbf4032a7c91b7ab8aeba4e 2024-11-20T06:55:07,199 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/table/dfbcaa3549b248128bd17f9f780aef93 2024-11-20T06:55:07,200 INFO [regionserver/61c2066a8a0f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:07,202 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-11-20T06:55:07,202 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T06:55:07,203 INFO [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T06:55:07,203 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732085707166Running coprocessor pre-close hooks at 1732085707166Disabling compacts and flushes for region at 1732085707166Disabling writes for close at 1732085707166Obtaining lock to block concurrent updates at 1732085707166Preparing flush snapshotting stores in 1588230740 at 1732085707166Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1732085707167 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732085707167Flushing 1588230740/info: creating writer at 1732085707167Flushing 1588230740/info: appending metadata at 1732085707190 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732085707190Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67396690: reopening flushed file at 1732085707193 (+3 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 31ms, sequenceid=18, compaction requested=false at 1732085707197 (+4 ms)Writing region close event to WAL at 1732085707199 (+2 ms)Running coprocessor post-close hooks at 1732085707202 (+3 ms)Closed at 1732085707203 (+1 ms) 2024-11-20T06:55:07,203 DEBUG [RS_CLOSE_META-regionserver/61c2066a8a0f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T06:55:07,212 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,46291,1732085706413] 2024-11-20T06:55:07,228 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,46291,1732085706413 already deleted, retry=false 2024-11-20T06:55:07,229 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,46291,1732085706413 expired; onlineServers=1 2024-11-20T06:55:07,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:07,312 INFO [RS:2;61c2066a8a0f:46291 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:55:07,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46291-0x101584ff29e0003, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:07,312 INFO [RS:2;61c2066a8a0f:46291 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,46291,1732085706413; zookeeper connection closed. 2024-11-20T06:55:07,313 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1319a3df {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1319a3df 2024-11-20T06:55:07,366 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(976): stopping server 61c2066a8a0f,43717,1732085702808; all regions closed. 2024-11-20T06:55:07,373 DEBUG [RS:0;61c2066a8a0f:43717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs 2024-11-20T06:55:07,373 INFO [RS:0;61c2066a8a0f:43717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C43717%2C1732085702808.meta:.meta(num 1732085706624) 2024-11-20T06:55:07,377 DEBUG [RS:0;61c2066a8a0f:43717 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/oldWALs 2024-11-20T06:55:07,377 INFO [RS:0;61c2066a8a0f:43717 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 61c2066a8a0f%2C43717%2C1732085702808:(num 1732085703537) 2024-11-20T06:55:07,377 DEBUG [RS:0;61c2066a8a0f:43717 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T06:55:07,377 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T06:55:07,377 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:55:07,377 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.ChoreService(370): Chore service for: regionserver/61c2066a8a0f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T06:55:07,378 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:55:07,378 INFO [regionserver/61c2066a8a0f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:55:07,378 INFO [RS:0;61c2066a8a0f:43717 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43717 2024-11-20T06:55:07,420 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:55:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/61c2066a8a0f,43717,1732085702808 2024-11-20T06:55:07,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T06:55:07,437 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [61c2066a8a0f,43717,1732085702808] 2024-11-20T06:55:07,453 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/61c2066a8a0f,43717,1732085702808 already deleted, retry=false 2024-11-20T06:55:07,453 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 61c2066a8a0f,43717,1732085702808 expired; onlineServers=0 2024-11-20T06:55:07,454 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '61c2066a8a0f,40491,1732085702631' ***** 2024-11-20T06:55:07,454 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T06:55:07,454 INFO [M:0;61c2066a8a0f:40491 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T06:55:07,454 INFO [M:0;61c2066a8a0f:40491 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T06:55:07,454 DEBUG [M:0;61c2066a8a0f:40491 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T06:55:07,454 DEBUG [M:0;61c2066a8a0f:40491 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T06:55:07,454 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T06:55:07,454 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085703273 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.large.0-1732085703273,5,FailOnTimeoutGroup] 2024-11-20T06:55:07,454 DEBUG [master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085703273 {}] cleaner.HFileCleaner(306): Exit Thread[master/61c2066a8a0f:0:becomeActiveMaster-HFileCleaner.small.0-1732085703273,5,FailOnTimeoutGroup] 2024-11-20T06:55:07,454 INFO [M:0;61c2066a8a0f:40491 {}] hbase.ChoreService(370): Chore service for: master/61c2066a8a0f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T06:55:07,454 INFO [M:0;61c2066a8a0f:40491 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T06:55:07,454 DEBUG [M:0;61c2066a8a0f:40491 {}] master.HMaster(1795): Stopping service threads 2024-11-20T06:55:07,455 INFO [M:0;61c2066a8a0f:40491 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T06:55:07,455 INFO [M:0;61c2066a8a0f:40491 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T06:55:07,455 INFO [M:0;61c2066a8a0f:40491 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T06:55:07,455 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T06:55:07,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T06:55:07,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T06:55:07,470 DEBUG [M:0;61c2066a8a0f:40491 {}] zookeeper.ZKUtil(347): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T06:55:07,470 WARN [M:0;61c2066a8a0f:40491 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T06:55:07,471 INFO [M:0;61c2066a8a0f:40491 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/.lastflushedseqids 2024-11-20T06:55:07,476 INFO [M:0;61c2066a8a0f:40491 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T06:55:07,477 INFO [M:0;61c2066a8a0f:40491 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T06:55:07,477 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T06:55:07,477 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:07,477 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:07,477 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T06:55:07,477 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:07,478 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.94 KB heapSize=44.16 KB 2024-11-20T06:55:07,486 DEBUG [M:0;61c2066a8a0f:40491 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5800e24ec0b74ae4b43e0aa0aaf39e49 2024-11-20T06:55:07,500 DEBUG [M:0;61c2066a8a0f:40491 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5800e24ec0b74ae4b43e0aa0aaf39e49 is 82, key is hbase:meta,,1/info:regioninfo/1732085706667/Put/seqid=0 2024-11-20T06:55:07,500 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5800e24ec0b74ae4b43e0aa0aaf39e49 2024-11-20T06:55:07,507 DEBUG [M:0;61c2066a8a0f:40491 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 35e1a65d2d114a909f7c74af3cc0f2fc 2024-11-20T06:55:07,520 DEBUG [M:0;61c2066a8a0f:40491 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35e1a65d2d114a909f7c74af3cc0f2fc is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732085704596/Put/seqid=0 2024-11-20T06:55:07,520 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.71 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35e1a65d2d114a909f7c74af3cc0f2fc 2024-11-20T06:55:07,527 DEBUG [M:0;61c2066a8a0f:40491 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2574f52380a642889bbeb61579e10d66 2024-11-20T06:55:07,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:07,537 INFO [RS:0;61c2066a8a0f:43717 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:55:07,537 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43717-0x101584ff29e0001, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:07,537 INFO [RS:0;61c2066a8a0f:43717 {}] regionserver.HRegionServer(1031): Exiting; stopping=61c2066a8a0f,43717,1732085702808; zookeeper connection closed. 2024-11-20T06:55:07,537 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e3dc72f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e3dc72f 2024-11-20T06:55:07,538 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-20T06:55:07,540 DEBUG [M:0;61c2066a8a0f:40491 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2574f52380a642889bbeb61579e10d66 is 69, key is 61c2066a8a0f,43717,1732085702808/rs:state/1732085703329/Put/seqid=0 2024-11-20T06:55:07,541 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2574f52380a642889bbeb61579e10d66 2024-11-20T06:55:07,542 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2574f52380a642889bbeb61579e10d66 2024-11-20T06:55:07,543 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5800e24ec0b74ae4b43e0aa0aaf39e49 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5800e24ec0b74ae4b43e0aa0aaf39e49 2024-11-20T06:55:07,544 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5800e24ec0b74ae4b43e0aa0aaf39e49, entries=14, sequenceid=90, filesize=5.9 K 2024-11-20T06:55:07,545 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/35e1a65d2d114a909f7c74af3cc0f2fc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35e1a65d2d114a909f7c74af3cc0f2fc 2024-11-20T06:55:07,546 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/35e1a65d2d114a909f7c74af3cc0f2fc, entries=11, sequenceid=90, filesize=7.8 K 2024-11-20T06:55:07,546 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2574f52380a642889bbeb61579e10d66 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2574f52380a642889bbeb61579e10d66 2024-11-20T06:55:07,548 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2574f52380a642889bbeb61579e10d66 2024-11-20T06:55:07,548 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d7eb59f5-7ad7-f0e0-54d5-158f786e23e3/298de4b1-e3a6-0d67-a2e2-5c774bdbaf63/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2574f52380a642889bbeb61579e10d66, entries=3, sequenceid=90, filesize=5.3 K 2024-11-20T06:55:07,548 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.94 KB/35780, heapSize ~43.87 KB/44920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=90, compaction requested=false 2024-11-20T06:55:07,550 INFO [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T06:55:07,550 DEBUG [M:0;61c2066a8a0f:40491 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732085707477Disabling compacts and flushes for region at 1732085707477Disabling writes for close at 1732085707477Obtaining lock to block concurrent updates at 1732085707478 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732085707478Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35780, getHeapSize=45160, getOffHeapSize=0, getCellsCount=111 at 1732085707478Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732085707478Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732085707478Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732085707499 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732085707500 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732085707502 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732085707520 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732085707520Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732085707522 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732085707539 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732085707540 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@62b9777f: reopening flushed file at 1732085707543 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e1789e7: reopening flushed file at 1732085707544 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@549dea4e: reopening flushed file at 1732085707546 (+2 ms)Finished flush of dataSize ~34.94 KB/35780, heapSize ~43.87 KB/44920, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=90, compaction requested=false at 1732085707548 (+2 ms)Writing region close event to WAL at 1732085707550 (+2 ms)Closed at 1732085707550 2024-11-20T06:55:07,551 INFO [M:0;61c2066a8a0f:40491 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T06:55:07,551 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T06:55:07,551 INFO [M:0;61c2066a8a0f:40491 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40491 2024-11-20T06:55:07,551 INFO [M:0;61c2066a8a0f:40491 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T06:55:07,679 INFO [M:0;61c2066a8a0f:40491 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T06:55:07,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:07,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40491-0x101584ff29e0000, quorum=127.0.0.1:61976, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T06:55:07,683 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T06:55:07,686 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-20T06:55:07,686 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T06:55:07,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=74 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Abort regionserver monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=383 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=376 (was 409), ProcessCount=11 (was 11), AvailableMemoryMB=4615 (was 4703)