2024-12-10 08:01:13,254 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 08:01:13,267 main DEBUG Took 0.010784 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 08:01:13,267 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 08:01:13,267 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 08:01:13,273 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 08:01:13,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,283 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 08:01:13,298 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,301 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,301 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,302 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,302 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,303 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,304 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,304 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,305 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,306 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,306 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,307 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,308 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,309 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,310 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,311 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,312 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 08:01:13,313 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,314 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 08:01:13,316 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 08:01:13,317 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 08:01:13,319 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 08:01:13,320 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 08:01:13,322 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 08:01:13,322 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 08:01:13,333 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 08:01:13,337 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 08:01:13,339 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 08:01:13,339 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 08:01:13,340 main DEBUG createAppenders(={Console}) 2024-12-10 08:01:13,341 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-10 08:01:13,341 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 08:01:13,341 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-10 08:01:13,342 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 08:01:13,342 main DEBUG OutputStream closed 2024-12-10 08:01:13,343 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 08:01:13,343 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 08:01:13,343 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-10 08:01:13,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 08:01:13,410 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 08:01:13,411 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 08:01:13,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 08:01:13,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 08:01:13,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 08:01:13,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 08:01:13,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 08:01:13,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 08:01:13,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 08:01:13,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 08:01:13,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 08:01:13,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 08:01:13,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 08:01:13,414 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 08:01:13,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 08:01:13,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 08:01:13,416 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 08:01:13,417 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 08:01:13,418 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-10 08:01:13,418 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 08:01:13,418 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-10T08:01:13,430 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-10 08:01:13,432 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 08:01:13,433 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T08:01:13,645 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f 2024-12-10T08:01:13,665 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=129, ProcessCount=11, AvailableMemoryMB=7471 2024-12-10T08:01:13,749 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T08:01:13,836 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:13,850 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/cluster_28c7b4dd-26ef-748c-8998-02c5f4bd8ad7, deleteOnExit=true 2024-12-10T08:01:13,981 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/cluster_28c7b4dd-26ef-748c-8998-02c5f4bd8ad7/zookeeper_0, clientPort=51160, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/cluster_28c7b4dd-26ef-748c-8998-02c5f4bd8ad7/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/cluster_28c7b4dd-26ef-748c-8998-02c5f4bd8ad7/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T08:01:13,992 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51160 2024-12-10T08:01:13,992 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:14,062 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 with version=8 2024-12-10T08:01:14,062 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/hbase-staging 2024-12-10T08:01:14,143 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T08:01:14,353 INFO [Time-limited test {}] client.ConnectionUtils(128): master/80fbf959555f:0 server-side Connection retries=45 2024-12-10T08:01:14,361 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:14,362 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:14,369 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T08:01:14,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:14,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T08:01:14,480 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T08:01:14,550 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T08:01:14,585 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T08:01:14,593 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T08:01:14,597 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T08:01:14,627 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 111645 (auto-detected) 2024-12-10T08:01:14,628 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T08:01:14,650 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44683 2024-12-10T08:01:14,677 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44683 connecting to ZooKeeper ensemble=127.0.0.1:51160 2024-12-10T08:01:14,794 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:446830x0, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T08:01:14,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44683-0x1000ecbe7810000 connected 2024-12-10T08:01:14,898 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:14,900 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:14,911 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:14,914 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4, hbase.cluster.distributed=false 2024-12-10T08:01:14,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T08:01:14,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44683 2024-12-10T08:01:14,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44683 2024-12-10T08:01:14,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44683 2024-12-10T08:01:14,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44683 2024-12-10T08:01:14,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44683 2024-12-10T08:01:15,009 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80fbf959555f:0 server-side Connection retries=45 2024-12-10T08:01:15,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:15,010 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:15,011 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T08:01:15,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:15,011 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T08:01:15,013 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T08:01:15,015 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T08:01:15,016 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40721 2024-12-10T08:01:15,018 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40721 connecting to ZooKeeper ensemble=127.0.0.1:51160 2024-12-10T08:01:15,018 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:15,019 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:15,033 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407210x0, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T08:01:15,034 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:407210x0, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:15,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40721-0x1000ecbe7810001 connected 2024-12-10T08:01:15,041 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T08:01:15,053 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.cache 2024-12-10T08:01:15,057 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T08:01:15,058 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T08:01:15,060 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T08:01:15,061 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T08:01:15,061 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T08:01:15,066 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T08:01:15,068 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T08:01:15,069 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.persistence is complete. 2024-12-10T08:01:15,072 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T08:01:15,074 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T08:01:15,078 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T08:01:15,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40721 2024-12-10T08:01:15,083 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40721 2024-12-10T08:01:15,084 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40721 2024-12-10T08:01:15,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40721 2024-12-10T08:01:15,085 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40721 2024-12-10T08:01:15,101 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80fbf959555f:0 server-side Connection retries=45 2024-12-10T08:01:15,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:15,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:15,101 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T08:01:15,101 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:15,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T08:01:15,102 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T08:01:15,102 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T08:01:15,103 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46651 2024-12-10T08:01:15,105 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46651 connecting to ZooKeeper ensemble=127.0.0.1:51160 2024-12-10T08:01:15,106 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:15,106 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:15,140 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:466510x0, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T08:01:15,141 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46651-0x1000ecbe7810002 connected 2024-12-10T08:01:15,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:15,142 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T08:01:15,143 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.cache 2024-12-10T08:01:15,143 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T08:01:15,144 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T08:01:15,144 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T08:01:15,146 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T08:01:15,146 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T08:01:15,146 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T08:01:15,147 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T08:01:15,147 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T08:01:15,147 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.persistence is complete. 2024-12-10T08:01:15,148 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T08:01:15,151 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T08:01:15,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46651 2024-12-10T08:01:15,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46651 2024-12-10T08:01:15,159 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46651 2024-12-10T08:01:15,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46651 2024-12-10T08:01:15,161 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46651 2024-12-10T08:01:15,181 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80fbf959555f:44683 2024-12-10T08:01:15,183 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/80fbf959555f,44683,1733817674202 2024-12-10T08:01:15,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:15,209 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:15,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:15,212 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80fbf959555f,44683,1733817674202 2024-12-10T08:01:15,242 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T08:01:15,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T08:01:15,243 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,244 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T08:01:15,245 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80fbf959555f,44683,1733817674202 from backup master directory 2024-12-10T08:01:15,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:15,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80fbf959555f,44683,1733817674202 2024-12-10T08:01:15,256 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:15,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:15,258 WARN [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T08:01:15,258 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80fbf959555f,44683,1733817674202 2024-12-10T08:01:15,260 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T08:01:15,262 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T08:01:15,271 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/hbase.id] with ID: 10eceb78-bb86-4e96-9bfc-891a417c9471 2024-12-10T08:01:15,271 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.tmp/hbase.id 2024-12-10T08:01:15,277 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/hbase.id] 2024-12-10T08:01:15,313 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:15,318 INFO [master/80fbf959555f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T08:01:15,337 INFO [master/80fbf959555f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-10T08:01:15,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,356 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,381 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T08:01:15,385 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T08:01:15,428 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T08:01:15,433 INFO [master/80fbf959555f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:15,479 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store 2024-12-10T08:01:15,496 INFO [master/80fbf959555f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T08:01:15,499 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:15,501 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T08:01:15,501 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:15,501 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:15,502 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T08:01:15,503 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:15,503 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:15,503 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733817675500Disabling compacts and flushes for region at 1733817675500Disabling writes for close at 1733817675503 (+3 ms)Writing region close event to WAL at 1733817675503Closed at 1733817675503 2024-12-10T08:01:15,504 WARN [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/.initializing 2024-12-10T08:01:15,504 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/WALs/80fbf959555f,44683,1733817674202 2024-12-10T08:01:15,516 INFO [master/80fbf959555f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T08:01:15,529 INFO [master/80fbf959555f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80fbf959555f%2C44683%2C1733817674202, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/WALs/80fbf959555f,44683,1733817674202, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/oldWALs, maxLogs=10 2024-12-10T08:01:15,563 INFO [master/80fbf959555f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/WALs/80fbf959555f,44683,1733817674202/80fbf959555f%2C44683%2C1733817674202.1733817675533 2024-12-10T08:01:15,564 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:15,565 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:15,565 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:15,569 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,571 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,606 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,637 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T08:01:15,643 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:15,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:15,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T08:01:15,650 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:15,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:15,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T08:01:15,655 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:15,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:15,656 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T08:01:15,660 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:15,661 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:15,662 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,666 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,667 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,671 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,672 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,676 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T08:01:15,679 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:15,685 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T08:01:15,686 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72022953, jitterRate=0.07322563230991364}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T08:01:15,691 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733817675583Initializing all the Stores at 1733817675583Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817675584 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817675584Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817675585 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817675585Cleaning up temporary data from old regions at 1733817675672 (+87 ms)Region opened successfully at 1733817675691 (+19 ms) 2024-12-10T08:01:15,693 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T08:01:15,722 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41ebde83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80fbf959555f/172.17.0.2:0 2024-12-10T08:01:15,752 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T08:01:15,763 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T08:01:15,763 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T08:01:15,766 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T08:01:15,767 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-10T08:01:15,772 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-10T08:01:15,773 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T08:01:15,798 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T08:01:15,808 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T08:01:15,858 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T08:01:15,861 INFO [master/80fbf959555f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T08:01:15,863 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T08:01:15,875 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T08:01:15,877 INFO [master/80fbf959555f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T08:01:15,881 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T08:01:15,891 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T08:01:15,893 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T08:01:15,900 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T08:01:15,920 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T08:01:15,931 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T08:01:15,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:15,950 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:15,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:15,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,951 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,953 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=80fbf959555f,44683,1733817674202, sessionid=0x1000ecbe7810000, setting cluster-up flag (Was=false) 2024-12-10T08:01:15,983 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:15,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:16,008 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T08:01:16,011 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80fbf959555f,44683,1733817674202 2024-12-10T08:01:16,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:16,034 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:16,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:16,058 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T08:01:16,061 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80fbf959555f,44683,1733817674202 2024-12-10T08:01:16,067 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T08:01:16,128 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:16,138 INFO [master/80fbf959555f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T08:01:16,144 INFO [master/80fbf959555f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T08:01:16,149 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80fbf959555f,44683,1733817674202 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T08:01:16,156 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:16,156 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:16,156 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:16,156 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:16,156 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80fbf959555f:0, corePoolSize=10, maxPoolSize=10 2024-12-10T08:01:16,157 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,157 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80fbf959555f:0, corePoolSize=2, maxPoolSize=2 2024-12-10T08:01:16,157 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,158 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733817706158 2024-12-10T08:01:16,160 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T08:01:16,161 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T08:01:16,161 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:16,161 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T08:01:16,163 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T08:01:16,163 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T08:01:16,164 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T08:01:16,164 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T08:01:16,164 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(746): ClusterId : 10eceb78-bb86-4e96-9bfc-891a417c9471 2024-12-10T08:01:16,164 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(746): ClusterId : 10eceb78-bb86-4e96-9bfc-891a417c9471 2024-12-10T08:01:16,165 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,166 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T08:01:16,167 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T08:01:16,167 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T08:01:16,165 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,168 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T08:01:16,169 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T08:01:16,170 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T08:01:16,172 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T08:01:16,172 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T08:01:16,174 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.large.0-1733817676174,5,FailOnTimeoutGroup] 2024-12-10T08:01:16,175 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.small.0-1733817676175,5,FailOnTimeoutGroup] 2024-12-10T08:01:16,176 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,176 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T08:01:16,177 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,177 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,186 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T08:01:16,186 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T08:01:16,186 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T08:01:16,186 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T08:01:16,192 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T08:01:16,192 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:16,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:16,200 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T08:01:16,204 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T08:01:16,204 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,204 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,205 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T08:01:16,208 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T08:01:16,208 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,209 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T08:01:16,209 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T08:01:16,209 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T08:01:16,210 DEBUG [RS:1;80fbf959555f:46651 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61877491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80fbf959555f/172.17.0.2:0 2024-12-10T08:01:16,210 DEBUG [RS:0;80fbf959555f:40721 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@567f4bf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80fbf959555f/172.17.0.2:0 2024-12-10T08:01:16,213 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T08:01:16,214 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,214 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T08:01:16,219 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T08:01:16,220 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,220 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T08:01:16,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:16,222 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:16,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T08:01:16,223 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T08:01:16,223 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T08:01:16,224 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80fbf959555f:40721 2024-12-10T08:01:16,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T08:01:16,226 DEBUG [RS:1;80fbf959555f:46651 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;80fbf959555f:46651 2024-12-10T08:01:16,228 INFO [RS:1;80fbf959555f:46651 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T08:01:16,228 INFO [RS:0;80fbf959555f:40721 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T08:01:16,229 INFO [RS:1;80fbf959555f:46651 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T08:01:16,229 INFO [RS:0;80fbf959555f:40721 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T08:01:16,229 DEBUG [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T08:01:16,229 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T08:01:16,232 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(2659): reportForDuty to master=80fbf959555f,44683,1733817674202 with port=46651, startcode=1733817675100 2024-12-10T08:01:16,232 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(2659): reportForDuty to master=80fbf959555f,44683,1733817674202 with port=40721, startcode=1733817674978 2024-12-10T08:01:16,233 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T08:01:16,234 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71052614, jitterRate=0.058766454458236694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T08:01:16,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733817676199Initializing all the Stores at 1733817676200 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817676200Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817676200Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817676200Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817676200Cleaning up temporary data from old regions at 1733817676223 (+23 ms)Region opened successfully at 1733817676237 (+14 ms) 2024-12-10T08:01:16,237 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T08:01:16,237 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T08:01:16,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T08:01:16,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T08:01:16,238 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T08:01:16,239 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:16,240 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733817676237Disabling compacts and flushes for region at 1733817676237Disabling writes for close at 1733817676238 (+1 ms)Writing region close event to WAL at 1733817676239 (+1 ms)Closed at 1733817676239 2024-12-10T08:01:16,242 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:16,243 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T08:01:16,243 DEBUG [RS:1;80fbf959555f:46651 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T08:01:16,243 DEBUG [RS:0;80fbf959555f:40721 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T08:01:16,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T08:01:16,259 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T08:01:16,261 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T08:01:16,307 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59283, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T08:01:16,307 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52561, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T08:01:16,315 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44683 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80fbf959555f,46651,1733817675100 2024-12-10T08:01:16,317 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44683 {}] master.ServerManager(517): Registering regionserver=80fbf959555f,46651,1733817675100 2024-12-10T08:01:16,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44683 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,326 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44683 {}] master.ServerManager(517): Registering regionserver=80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,330 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:16,330 DEBUG [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:16,330 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T08:01:16,330 DEBUG [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T08:01:16,330 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T08:01:16,330 DEBUG [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T08:01:16,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:16,391 DEBUG [RS:1;80fbf959555f:46651 {}] zookeeper.ZKUtil(111): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80fbf959555f,46651,1733817675100 2024-12-10T08:01:16,391 DEBUG [RS:0;80fbf959555f:40721 {}] zookeeper.ZKUtil(111): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,391 WARN [RS:0;80fbf959555f:40721 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T08:01:16,391 WARN [RS:1;80fbf959555f:46651 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T08:01:16,392 INFO [RS:0;80fbf959555f:40721 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:16,392 INFO [RS:1;80fbf959555f:46651 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:16,392 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,392 DEBUG [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46651,1733817675100 2024-12-10T08:01:16,396 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80fbf959555f,46651,1733817675100] 2024-12-10T08:01:16,396 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80fbf959555f,40721,1733817674978] 2024-12-10T08:01:16,412 WARN [80fbf959555f:44683 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T08:01:16,417 INFO [RS:1;80fbf959555f:46651 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T08:01:16,417 INFO [RS:0;80fbf959555f:40721 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T08:01:16,430 INFO [RS:0;80fbf959555f:40721 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T08:01:16,430 INFO [RS:1;80fbf959555f:46651 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T08:01:16,435 INFO [RS:0;80fbf959555f:40721 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T08:01:16,435 INFO [RS:1;80fbf959555f:46651 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T08:01:16,435 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,435 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,436 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T08:01:16,436 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T08:01:16,440 INFO [RS:0;80fbf959555f:40721 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T08:01:16,440 INFO [RS:1;80fbf959555f:46651 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T08:01:16,442 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,442 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,442 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,442 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,442 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,442 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,442 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,442 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,442 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80fbf959555f:0, corePoolSize=2, maxPoolSize=2 2024-12-10T08:01:16,443 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80fbf959555f:0, corePoolSize=2, maxPoolSize=2 2024-12-10T08:01:16,443 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,443 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,444 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,444 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,444 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:16,444 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,444 DEBUG [RS:0;80fbf959555f:40721 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:16,444 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,444 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:16,444 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:16,445 DEBUG [RS:1;80fbf959555f:46651 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:16,447 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,447 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,447 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,447 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,447 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,447 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,447 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,447 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,448 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,448 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,448 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,40721,1733817674978-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T08:01:16,448 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,46651,1733817675100-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T08:01:16,473 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T08:01:16,473 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T08:01:16,475 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,40721,1733817674978-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,475 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,46651,1733817675100-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,476 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,476 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,476 INFO [RS:0;80fbf959555f:40721 {}] regionserver.Replication(171): 80fbf959555f,40721,1733817674978 started 2024-12-10T08:01:16,476 INFO [RS:1;80fbf959555f:46651 {}] regionserver.Replication(171): 80fbf959555f,46651,1733817675100 started 2024-12-10T08:01:16,493 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,493 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:16,493 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1482): Serving as 80fbf959555f,40721,1733817674978, RpcServer on 80fbf959555f/172.17.0.2:40721, sessionid=0x1000ecbe7810001 2024-12-10T08:01:16,493 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(1482): Serving as 80fbf959555f,46651,1733817675100, RpcServer on 80fbf959555f/172.17.0.2:46651, sessionid=0x1000ecbe7810002 2024-12-10T08:01:16,494 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T08:01:16,494 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T08:01:16,494 DEBUG [RS:0;80fbf959555f:40721 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,494 DEBUG [RS:1;80fbf959555f:46651 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80fbf959555f,46651,1733817675100 2024-12-10T08:01:16,494 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,40721,1733817674978' 2024-12-10T08:01:16,494 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,46651,1733817675100' 2024-12-10T08:01:16,495 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T08:01:16,495 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T08:01:16,496 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T08:01:16,496 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T08:01:16,496 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T08:01:16,496 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T08:01:16,497 DEBUG [RS:0;80fbf959555f:40721 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,497 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T08:01:16,497 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,40721,1733817674978' 2024-12-10T08:01:16,497 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T08:01:16,497 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T08:01:16,497 DEBUG [RS:1;80fbf959555f:46651 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80fbf959555f,46651,1733817675100 2024-12-10T08:01:16,497 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,46651,1733817675100' 2024-12-10T08:01:16,497 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T08:01:16,498 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T08:01:16,498 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T08:01:16,498 DEBUG [RS:0;80fbf959555f:40721 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T08:01:16,498 DEBUG [RS:1;80fbf959555f:46651 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T08:01:16,498 INFO [RS:0;80fbf959555f:40721 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T08:01:16,498 INFO [RS:1;80fbf959555f:46651 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T08:01:16,498 INFO [RS:1;80fbf959555f:46651 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T08:01:16,498 INFO [RS:0;80fbf959555f:40721 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T08:01:16,606 INFO [RS:1;80fbf959555f:46651 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T08:01:16,606 INFO [RS:0;80fbf959555f:40721 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T08:01:16,608 INFO [RS:1;80fbf959555f:46651 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C46651%2C1733817675100, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46651,1733817675100, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:16,608 INFO [RS:0;80fbf959555f:40721 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C40721%2C1733817674978, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,40721,1733817674978, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:16,628 INFO [RS:0;80fbf959555f:40721 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,40721,1733817674978/80fbf959555f%2C40721%2C1733817674978.1733817676613 2024-12-10T08:01:16,629 DEBUG [RS:0;80fbf959555f:40721 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:16,632 INFO [RS:1;80fbf959555f:46651 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46651,1733817675100/80fbf959555f%2C46651%2C1733817675100.1733817676614 2024-12-10T08:01:16,632 DEBUG [RS:1;80fbf959555f:46651 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:16,665 DEBUG [80fbf959555f:44683 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T08:01:16,677 DEBUG [80fbf959555f:44683 {}] balancer.BalancerClusterState(204): Hosts are {80fbf959555f=0} racks are {/default-rack=0} 2024-12-10T08:01:16,680 DEBUG [80fbf959555f:44683 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T08:01:16,680 DEBUG [80fbf959555f:44683 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T08:01:16,680 DEBUG [80fbf959555f:44683 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T08:01:16,680 DEBUG [80fbf959555f:44683 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T08:01:16,680 INFO [80fbf959555f:44683 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T08:01:16,680 INFO [80fbf959555f:44683 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T08:01:16,680 DEBUG [80fbf959555f:44683 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T08:01:16,686 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,690 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80fbf959555f,40721,1733817674978, state=OPENING 2024-12-10T08:01:16,716 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T08:01:16,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:16,725 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:16,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:16,726 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:16,727 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:16,727 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:16,729 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T08:01:16,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=80fbf959555f,40721,1733817674978}] 2024-12-10T08:01:16,912 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T08:01:16,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43223, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T08:01:16,923 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T08:01:16,924 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:16,925 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T08:01:16,926 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C40721%2C1733817674978.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,40721,1733817674978, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:16,942 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,40721,1733817674978/80fbf959555f%2C40721%2C1733817674978.meta.1733817676927.meta 2024-12-10T08:01:16,943 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:16,943 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:16,944 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T08:01:16,946 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T08:01:16,951 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T08:01:16,954 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T08:01:16,955 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:16,955 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T08:01:16,955 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T08:01:16,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T08:01:16,958 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T08:01:16,958 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T08:01:16,959 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T08:01:16,959 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T08:01:16,960 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T08:01:16,960 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,960 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T08:01:16,961 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T08:01:16,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:16,961 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:16,961 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T08:01:16,962 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:16,963 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:16,963 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T08:01:16,963 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T08:01:16,963 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T08:01:16,964 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T08:01:16,965 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62462907, jitterRate=-0.06923015415668488}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T08:01:16,965 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T08:01:16,967 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733817676956Writing region info on filesystem at 1733817676956Initializing all the Stores at 1733817676956Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817676956Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817676956Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817676957 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817676957Cleaning up temporary data from old regions at 1733817676963 (+6 ms)Running coprocessor post-open hooks at 1733817676965 (+2 ms)Region opened successfully at 1733817676967 (+2 ms) 2024-12-10T08:01:16,972 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733817676902 2024-12-10T08:01:16,980 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T08:01:16,981 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T08:01:16,982 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80fbf959555f,40721,1733817674978 2024-12-10T08:01:16,983 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80fbf959555f,40721,1733817674978, state=OPEN 2024-12-10T08:01:17,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:17,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:17,059 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:17,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:17,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:17,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:17,060 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=80fbf959555f,40721,1733817674978 2024-12-10T08:01:17,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T08:01:17,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=80fbf959555f,40721,1733817674978 in 329 msec 2024-12-10T08:01:17,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T08:01:17,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 818 msec 2024-12-10T08:01:17,073 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:17,073 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T08:01:17,089 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T08:01:17,090 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80fbf959555f,40721,1733817674978, seqNum=-1] 2024-12-10T08:01:17,106 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T08:01:17,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41911, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T08:01:17,125 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0300 sec 2024-12-10T08:01:17,125 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733817677125, completionTime=-1 2024-12-10T08:01:17,129 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T08:01:17,129 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T08:01:17,150 INFO [master/80fbf959555f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T08:01:17,150 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733817737150 2024-12-10T08:01:17,150 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733817797150 2024-12-10T08:01:17,150 INFO [master/80fbf959555f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-10T08:01:17,153 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T08:01:17,160 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,44683,1733817674202-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:17,161 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,44683,1733817674202-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:17,161 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,44683,1733817674202-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:17,163 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80fbf959555f:44683, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:17,163 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:17,164 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:17,170 DEBUG [master/80fbf959555f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T08:01:17,188 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.930sec 2024-12-10T08:01:17,189 INFO [master/80fbf959555f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T08:01:17,190 INFO [master/80fbf959555f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T08:01:17,191 INFO [master/80fbf959555f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T08:01:17,192 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T08:01:17,192 INFO [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T08:01:17,192 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,44683,1733817674202-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T08:01:17,193 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,44683,1733817674202-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T08:01:17,194 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T08:01:17,194 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T08:01:17,195 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,44683,1733817674202-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:17,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T08:01:17,278 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T08:01:17,278 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T08:01:17,282 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80fbf959555f,44683,-1 for getting cluster id 2024-12-10T08:01:17,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T08:01:17,291 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '10eceb78-bb86-4e96-9bfc-891a417c9471' 2024-12-10T08:01:17,293 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T08:01:17,293 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "10eceb78-bb86-4e96-9bfc-891a417c9471" 2024-12-10T08:01:17,295 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49244511, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T08:01:17,295 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80fbf959555f,44683,-1] 2024-12-10T08:01:17,297 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T08:01:17,299 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:17,300 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T08:01:17,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55969d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T08:01:17,303 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T08:01:17,309 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80fbf959555f,40721,1733817674978, seqNum=-1] 2024-12-10T08:01:17,309 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T08:01:17,312 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T08:01:17,332 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=80fbf959555f,44683,1733817674202 2024-12-10T08:01:17,337 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T08:01:17,341 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 80fbf959555f,44683,1733817674202 2024-12-10T08:01:17,343 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@12b12a2a 2024-12-10T08:01:17,343 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T08:01:17,346 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T08:01:17,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733817677332', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T08:01:17,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 2024-12-10T08:01:17,370 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T08:01:17,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733817677332" procId is: 4 2024-12-10T08:01:17,373 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:17,374 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T08:01:17,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:17,390 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9e905d9bcfa5f4772ca9d1825ee8f694, NAME => 'testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733817677332', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:17,396 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:17,397 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(1722): Closing 9e905d9bcfa5f4772ca9d1825ee8f694, disabling compactions & flushes 2024-12-10T08:01:17,397 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:17,397 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:17,397 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. after waiting 0 ms 2024-12-10T08:01:17,397 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:17,397 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:17,397 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733817677332-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9e905d9bcfa5f4772ca9d1825ee8f694: Waiting for close lock at 1733817677397Disabling compacts and flushes for region at 1733817677397Disabling writes for close at 1733817677397Writing region close event to WAL at 1733817677397Closed at 1733817677397 2024-12-10T08:01:17,399 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T08:01:17,403 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733817677399"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733817677399"}]},"ts":"1733817677399"} 2024-12-10T08:01:17,407 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T08:01:17,408 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T08:01:17,410 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733817677332","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733817677408"}]},"ts":"1733817677408"} 2024-12-10T08:01:17,413 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733817677332, state=ENABLING in hbase:meta 2024-12-10T08:01:17,414 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {80fbf959555f=0} racks are {/default-rack=0} 2024-12-10T08:01:17,415 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T08:01:17,415 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T08:01:17,415 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T08:01:17,415 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T08:01:17,415 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T08:01:17,415 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T08:01:17,415 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T08:01:17,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, ASSIGN}] 2024-12-10T08:01:17,418 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, ASSIGN 2024-12-10T08:01:17,419 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, ASSIGN; state=OFFLINE, location=80fbf959555f,46651,1733817675100; forceNewPlan=false, retain=false 2024-12-10T08:01:17,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:17,575 INFO [80fbf959555f:44683 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T08:01:17,576 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e905d9bcfa5f4772ca9d1825ee8f694, regionState=OPENING, regionLocation=80fbf959555f,46651,1733817675100 2024-12-10T08:01:17,585 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, ASSIGN because future has completed 2024-12-10T08:01:17,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,46651,1733817675100}] 2024-12-10T08:01:17,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:17,740 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T08:01:17,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48323, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T08:01:17,749 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:17,749 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e905d9bcfa5f4772ca9d1825ee8f694, NAME => 'testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:17,750 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733817677332 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,750 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:17,750 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,751 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,752 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,756 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e905d9bcfa5f4772ca9d1825ee8f694 columnFamilyName family 2024-12-10T08:01:17,756 DEBUG [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:17,756 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.HStore(327): Store=9e905d9bcfa5f4772ca9d1825ee8f694/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:17,757 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,758 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,758 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,758 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,758 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,759 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,765 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T08:01:17,765 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9e905d9bcfa5f4772ca9d1825ee8f694; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74677821, jitterRate=0.11278624832630157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T08:01:17,765 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:17,766 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9e905d9bcfa5f4772ca9d1825ee8f694: Running coprocessor pre-open hook at 1733817677751Writing region info on filesystem at 1733817677751Initializing all the Stores at 1733817677751Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817677751Cleaning up temporary data from old regions at 1733817677758 (+7 ms)Running coprocessor post-open hooks at 1733817677765 (+7 ms)Region opened successfully at 1733817677766 (+1 ms) 2024-12-10T08:01:17,768 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694., pid=6, masterSystemTime=1733817677739 2024-12-10T08:01:17,771 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9e905d9bcfa5f4772ca9d1825ee8f694, regionState=OPEN, openSeqNum=2, regionLocation=80fbf959555f,46651,1733817675100 2024-12-10T08:01:17,772 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:17,772 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:17,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,46651,1733817675100 because future has completed 2024-12-10T08:01:17,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T08:01:17,778 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,46651,1733817675100 in 190 msec 2024-12-10T08:01:17,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T08:01:17,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, ASSIGN in 362 msec 2024-12-10T08:01:17,780 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T08:01:17,781 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733817677332","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733817677780"}]},"ts":"1733817677780"} 2024-12-10T08:01:17,783 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733817677332, state=ENABLED in hbase:meta 2024-12-10T08:01:17,784 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T08:01:17,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733817677332 in 420 msec 2024-12-10T08:01:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:18,009 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733817677332 completed 2024-12-10T08:01:18,009 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733817677332 get assigned. Timeout = 60000ms 2024-12-10T08:01:18,012 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T08:01:18,018 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733817677332 assigned to meta. Checking AM states. 2024-12-10T08:01:18,019 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T08:01:18,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733817677332 assigned. 2024-12-10T08:01:18,026 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733817677332', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694., hostname=80fbf959555f,46651,1733817675100, seqNum=2] 2024-12-10T08:01:18,028 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T08:01:18,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43994, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T08:01:18,039 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9e905d9bcfa5f4772ca9d1825ee8f694 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T08:01:18,059 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:18,089 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/.tmp/family/6dd84d9992504b149ef2f4d4b8ccb510 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T08:01:18,093 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/.tmp/family/6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:18,124 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/.tmp/family/6dd84d9992504b149ef2f4d4b8ccb510 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:18,125 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510, delay=949 ms 2024-12-10T08:01:18,127 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T08:01:18,134 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9e905d9bcfa5f4772ca9d1825ee8f694 in 93ms, sequenceid=6, compaction requested=false 2024-12-10T08:01:18,135 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733817677332' 2024-12-10T08:01:18,138 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9e905d9bcfa5f4772ca9d1825ee8f694: 2024-12-10T08:01:19,077 DEBUG [hfile-prefetch-1733817678125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6dd84d9992504b149ef2f4d4b8ccb510_0. 2024-12-10T08:01:19,083 DEBUG [hfile-prefetch-1733817678125 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6dd84d9992504b149ef2f4d4b8ccb510_117. 2024-12-10T08:01:19,084 DEBUG [hfile-prefetch-1733817678125 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510, with total blocks 2, and data blocks 1 2024-12-10T08:01:19,084 DEBUG [hfile-prefetch-1733817678125 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T08:01:19,085 DEBUG [hfile-prefetch-1733817678125 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T08:01:19,085 DEBUG [hfile-prefetch-1733817678125 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510. Will sleep 100ms and try the verification again. 2024-12-10T08:01:19,153 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9e905d9bcfa5f4772ca9d1825ee8f694, source=80fbf959555f,46651,1733817675100, destination=80fbf959555f,40721,1733817674978, warming up region on 80fbf959555f,40721,1733817674978 2024-12-10T08:01:19,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9e905d9bcfa5f4772ca9d1825ee8f694, source=80fbf959555f,46651,1733817675100, destination=80fbf959555f,40721,1733817674978, running balancer 2024-12-10T08:01:19,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, REOPEN/MOVE 2024-12-10T08:01:19,156 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, REOPEN/MOVE 2024-12-10T08:01:19,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9e905d9bcfa5f4772ca9d1825ee8f694, NAME => 'testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:19,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:19,157 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e905d9bcfa5f4772ca9d1825ee8f694, regionState=CLOSING, regionLocation=80fbf959555f,46651,1733817675100 2024-12-10T08:01:19,157 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,158 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e905d9bcfa5f4772ca9d1825ee8f694 columnFamilyName family 2024-12-10T08:01:19,158 DEBUG [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:19,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, REOPEN/MOVE because future has completed 2024-12-10T08:01:19,161 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-10T08:01:19,162 DEBUG [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:19,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,46651,1733817675100}] 2024-12-10T08:01:19,162 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.HStore(327): Store=9e905d9bcfa5f4772ca9d1825ee8f694/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(1722): Closing 9e905d9bcfa5f4772ca9d1825ee8f694, disabling compactions & flushes 2024-12-10T08:01:19,162 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. after waiting 0 ms 2024-12-10T08:01:19,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,163 DEBUG [StoreFileCloser-9e905d9bcfa5f4772ca9d1825ee8f694-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:19,164 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40721 {}] regionserver.HRegion(1676): Region close journal for 9e905d9bcfa5f4772ca9d1825ee8f694: Waiting for close lock at 1733817679162Disabling compacts and flushes for region at 1733817679162Disabling writes for close at 1733817679162Writing region close event to WAL at 1733817679164 (+2 ms)Closed at 1733817679164 2024-12-10T08:01:19,165 WARN [hfile-prefetch-1733817678125 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-10T08:01:19,166 DEBUG [hfile-prefetch-1733817678125 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44683 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-10T08:01:19,249 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T08:01:19,249 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T08:01:19,250 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:19,250 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:19,264 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T08:01:19,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T08:01:19,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:19,264 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:19,268 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:19,268 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817679264 2024-12-10T08:01:19,268 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T08:01:19,271 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-10T08:01:19,271 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733817679249 2024-12-10T08:01:19,271 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T08:01:19,322 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,323 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-10T08:01:19,324 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9e905d9bcfa5f4772ca9d1825ee8f694, disabling compactions & flushes 2024-12-10T08:01:19,324 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,324 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,324 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. after waiting 0 ms 2024-12-10T08:01:19,324 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,327 DEBUG [StoreFileCloser-9e905d9bcfa5f4772ca9d1825ee8f694-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 6dd84d9992504b149ef2f4d4b8ccb510, starting offset: 0, end offset: 9223372036854775807 2024-12-10T08:01:19,329 DEBUG [StoreFileCloser-9e905d9bcfa5f4772ca9d1825ee8f694-family-1 {}] bucket.BucketCache(856): removed key 6dd84d9992504b149ef2f4d4b8ccb510_0 from back map with offset lock 2094080 in the evict process 2024-12-10T08:01:19,337 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T08:01:19,339 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,339 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9e905d9bcfa5f4772ca9d1825ee8f694: Waiting for close lock at 1733817679324Running coprocessor pre-close hooks at 1733817679324Disabling compacts and flushes for region at 1733817679324Disabling writes for close at 1733817679324Writing region close event to WAL at 1733817679330 (+6 ms)Running coprocessor post-close hooks at 1733817679337 (+7 ms)Closed at 1733817679338 (+1 ms) 2024-12-10T08:01:19,339 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9e905d9bcfa5f4772ca9d1825ee8f694 move to 80fbf959555f,40721,1733817674978 record at close sequenceid=6 2024-12-10T08:01:19,343 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,344 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e905d9bcfa5f4772ca9d1825ee8f694, regionState=CLOSED 2024-12-10T08:01:19,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,46651,1733817675100 because future has completed 2024-12-10T08:01:19,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T08:01:19,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,46651,1733817675100 in 188 msec 2024-12-10T08:01:19,352 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, REOPEN/MOVE; state=CLOSED, location=80fbf959555f,40721,1733817674978; forceNewPlan=false, retain=false 2024-12-10T08:01:19,368 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T08:01:19,369 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T08:01:19,369 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:19,369 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:19,371 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T08:01:19,371 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T08:01:19,371 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:19,371 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:19,372 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:19,372 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817679369 2024-12-10T08:01:19,372 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T08:01:19,375 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:19,375 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817679371 2024-12-10T08:01:19,375 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T08:01:19,502 INFO [80fbf959555f:44683 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T08:01:19,503 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e905d9bcfa5f4772ca9d1825ee8f694, regionState=OPENING, regionLocation=80fbf959555f,40721,1733817674978 2024-12-10T08:01:19,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, REOPEN/MOVE because future has completed 2024-12-10T08:01:19,506 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,40721,1733817674978}] 2024-12-10T08:01:19,666 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,666 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9e905d9bcfa5f4772ca9d1825ee8f694, NAME => 'testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:19,667 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733817677332 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,667 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:19,667 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,667 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,668 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,669 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9e905d9bcfa5f4772ca9d1825ee8f694 columnFamilyName family 2024-12-10T08:01:19,670 DEBUG [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:19,672 DEBUG [StoreFileOpener-9e905d9bcfa5f4772ca9d1825ee8f694-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510, delay=980 ms 2024-12-10T08:01:19,674 DEBUG [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:19,674 INFO [StoreOpener-9e905d9bcfa5f4772ca9d1825ee8f694-1 {}] regionserver.HStore(327): Store=9e905d9bcfa5f4772ca9d1825ee8f694/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:19,674 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,675 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,676 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,676 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,676 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,680 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,681 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9e905d9bcfa5f4772ca9d1825ee8f694; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69781571, jitterRate=0.03982643783092499}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T08:01:19,681 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,682 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9e905d9bcfa5f4772ca9d1825ee8f694: Running coprocessor pre-open hook at 1733817679668Writing region info on filesystem at 1733817679668Initializing all the Stores at 1733817679668Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817679668Cleaning up temporary data from old regions at 1733817679676 (+8 ms)Running coprocessor post-open hooks at 1733817679681 (+5 ms)Region opened successfully at 1733817679682 (+1 ms) 2024-12-10T08:01:19,682 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694., pid=9, masterSystemTime=1733817679659 2024-12-10T08:01:19,685 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,685 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,685 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9e905d9bcfa5f4772ca9d1825ee8f694, regionState=OPEN, openSeqNum=10, regionLocation=80fbf959555f,40721,1733817674978 2024-12-10T08:01:19,688 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,40721,1733817674978 because future has completed 2024-12-10T08:01:19,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-10T08:01:19,691 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9e905d9bcfa5f4772ca9d1825ee8f694, server=80fbf959555f,40721,1733817674978 in 183 msec 2024-12-10T08:01:19,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733817677332, region=9e905d9bcfa5f4772ca9d1825ee8f694, REOPEN/MOVE in 537 msec 2024-12-10T08:01:19,708 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T08:01:19,709 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T08:01:19,709 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:19,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:19,713 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:19,713 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T08:01:19,713 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T08:01:19,713 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-12-10T08:01:19,714 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=80fbf959555f,44683,1733817674202 2024-12-10T08:01:19,741 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:19,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:19,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:19,741 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:19,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:19,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T08:01:19,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:19,743 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T08:01:19,743 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:19,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:19,743 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:19,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:19,744 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80fbf959555f,40721,1733817674978' ***** 2024-12-10T08:01:19,744 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:19,744 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T08:01:19,744 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80fbf959555f,46651,1733817675100' ***** 2024-12-10T08:01:19,744 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T08:01:19,745 INFO [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T08:01:19,745 INFO [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T08:01:19,745 INFO [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.persistence 2024-12-10T08:01:19,745 INFO [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/bucket.persistence 2024-12-10T08:01:19,745 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T08:01:19,745 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T08:01:19,745 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T08:01:19,746 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T08:01:19,745 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T08:01:19,746 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T08:01:19,746 INFO [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T08:01:19,746 INFO [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T08:01:19,746 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T08:01:19,746 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T08:01:19,750 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T08:01:19,750 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T08:01:19,751 INFO [regionserver/80fbf959555f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:19,756 DEBUG [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1487): Thread RS:0;80fbf959555f:40721 started persisting bucket cache to file 2024-12-10T08:01:19,756 DEBUG [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:19,756 DEBUG [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:19,756 DEBUG [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1487): Thread RS:1;80fbf959555f:46651 started persisting bucket cache to file 2024-12-10T08:01:19,756 DEBUG [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:19,756 DEBUG [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:19,759 DEBUG [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:19,759 DEBUG [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:19,759 DEBUG [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817679756 2024-12-10T08:01:19,759 DEBUG [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817679756 2024-12-10T08:01:19,759 DEBUG [RS:0;80fbf959555f:40721 {}] bucket.BucketCache(1509): Thread RS:0;80fbf959555f:40721 finished persisting bucket cache to file, renaming 2024-12-10T08:01:19,759 DEBUG [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1509): Thread RS:1;80fbf959555f:46651 finished persisting bucket cache to file, renaming 2024-12-10T08:01:19,759 WARN [RS:1;80fbf959555f:46651 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T08:01:19,759 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T08:01:19,759 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T08:01:19,759 INFO [RS:0;80fbf959555f:40721 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T08:01:19,759 INFO [RS:1;80fbf959555f:46651 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T08:01:19,759 INFO [RS:0;80fbf959555f:40721 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T08:01:19,759 INFO [RS:1;80fbf959555f:46651 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T08:01:19,759 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(3091): Received CLOSE for 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,759 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(959): stopping server 80fbf959555f,46651,1733817675100 2024-12-10T08:01:19,760 INFO [RS:1;80fbf959555f:46651 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T08:01:19,760 INFO [RS:1;80fbf959555f:46651 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;80fbf959555f:46651. 2024-12-10T08:01:19,760 DEBUG [RS:1;80fbf959555f:46651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:19,760 DEBUG [RS:1;80fbf959555f:46651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:19,760 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(976): stopping server 80fbf959555f,46651,1733817675100; all regions closed. 2024-12-10T08:01:19,760 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(959): stopping server 80fbf959555f,40721,1733817674978 2024-12-10T08:01:19,760 INFO [RS:0;80fbf959555f:40721 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T08:01:19,760 INFO [RS:0;80fbf959555f:40721 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;80fbf959555f:40721. 2024-12-10T08:01:19,760 DEBUG [RS:0;80fbf959555f:40721 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:19,760 DEBUG [RS:0;80fbf959555f:40721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:19,760 INFO [RS:0;80fbf959555f:40721 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T08:01:19,760 INFO [RS:0;80fbf959555f:40721 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T08:01:19,761 INFO [RS:0;80fbf959555f:40721 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T08:01:19,761 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T08:01:19,761 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T08:01:19,761 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9e905d9bcfa5f4772ca9d1825ee8f694, disabling compactions & flushes 2024-12-10T08:01:19,761 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,761 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T08:01:19,761 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,761 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1325): Online Regions={9e905d9bcfa5f4772ca9d1825ee8f694=testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694., 1588230740=hbase:meta,,1.1588230740} 2024-12-10T08:01:19,761 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T08:01:19,761 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. after waiting 0 ms 2024-12-10T08:01:19,761 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T08:01:19,761 INFO [regionserver/80fbf959555f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:19,761 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,761 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T08:01:19,761 DEBUG [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 9e905d9bcfa5f4772ca9d1825ee8f694 2024-12-10T08:01:19,761 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T08:01:19,762 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T08:01:19,762 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T08:01:19,762 DEBUG [StoreFileCloser-9e905d9bcfa5f4772ca9d1825ee8f694-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/family/6dd84d9992504b149ef2f4d4b8ccb510 2024-12-10T08:01:19,762 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-10T08:01:19,765 DEBUG [RS:1;80fbf959555f:46651 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:19,765 INFO [RS:1;80fbf959555f:46651 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C46651%2C1733817675100:(num 1733817676614) 2024-12-10T08:01:19,765 DEBUG [RS:1;80fbf959555f:46651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:19,765 INFO [RS:1;80fbf959555f:46651 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:19,766 INFO [RS:1;80fbf959555f:46651 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T08:01:19,766 INFO [RS:1;80fbf959555f:46651 {}] hbase.ChoreService(370): Chore service for: regionserver/80fbf959555f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T08:01:19,766 INFO [RS:1;80fbf959555f:46651 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T08:01:19,766 INFO [regionserver/80fbf959555f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T08:01:19,766 INFO [RS:1;80fbf959555f:46651 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T08:01:19,767 INFO [RS:1;80fbf959555f:46651 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T08:01:19,767 INFO [RS:1;80fbf959555f:46651 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T08:01:19,767 INFO [RS:1;80fbf959555f:46651 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46651 2024-12-10T08:01:19,768 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnRegionMove1733817677332/9e905d9bcfa5f4772ca9d1825ee8f694/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-10T08:01:19,769 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,769 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9e905d9bcfa5f4772ca9d1825ee8f694: Waiting for close lock at 1733817679760Running coprocessor pre-close hooks at 1733817679761 (+1 ms)Disabling compacts and flushes for region at 1733817679761Disabling writes for close at 1733817679761Writing region close event to WAL at 1733817679763 (+2 ms)Running coprocessor post-close hooks at 1733817679769 (+6 ms)Closed at 1733817679769 2024-12-10T08:01:19,769 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694. 2024-12-10T08:01:19,775 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 62fcd0cf725e4dc59434661df32df724 2024-12-10T08:01:19,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:19,789 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80fbf959555f,46651,1733817675100 2024-12-10T08:01:19,789 INFO [RS:1;80fbf959555f:46651 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T08:01:19,792 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/62fcd0cf725e4dc59434661df32df724 is 205, key is testBlockEvictionOnRegionMove1733817677332,,1733817677347.9e905d9bcfa5f4772ca9d1825ee8f694./info:regioninfo/1733817679685/Put/seqid=0 2024-12-10T08:01:19,793 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/62fcd0cf725e4dc59434661df32df724 2024-12-10T08:01:19,799 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80fbf959555f,46651,1733817675100] 2024-12-10T08:01:19,801 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0452b6421865453a9c96a5cf69151bfa 2024-12-10T08:01:19,807 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80fbf959555f,46651,1733817675100 already deleted, retry=false 2024-12-10T08:01:19,807 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80fbf959555f,46651,1733817675100 expired; onlineServers=1 2024-12-10T08:01:19,813 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/ns/0452b6421865453a9c96a5cf69151bfa is 43, key is default/ns:d/1733817677112/Put/seqid=0 2024-12-10T08:01:19,813 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/ns/0452b6421865453a9c96a5cf69151bfa 2024-12-10T08:01:19,820 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9fa082f6f44248ddbcd71895e4aa036b 2024-12-10T08:01:19,834 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/table/9fa082f6f44248ddbcd71895e4aa036b is 78, key is testBlockEvictionOnRegionMove1733817677332/table:state/1733817677780/Put/seqid=0 2024-12-10T08:01:19,835 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/table/9fa082f6f44248ddbcd71895e4aa036b 2024-12-10T08:01:19,838 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/62fcd0cf725e4dc59434661df32df724 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/62fcd0cf725e4dc59434661df32df724 2024-12-10T08:01:19,839 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/62fcd0cf725e4dc59434661df32df724, delay=965 ms 2024-12-10T08:01:19,840 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/62fcd0cf725e4dc59434661df32df724, entries=15, sequenceid=15, filesize=7.6 K 2024-12-10T08:01:19,840 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/ns/0452b6421865453a9c96a5cf69151bfa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/0452b6421865453a9c96a5cf69151bfa 2024-12-10T08:01:19,841 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/0452b6421865453a9c96a5cf69151bfa, delay=962 ms 2024-12-10T08:01:19,843 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/0452b6421865453a9c96a5cf69151bfa, entries=2, sequenceid=15, filesize=5.0 K 2024-12-10T08:01:19,843 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/table/9fa082f6f44248ddbcd71895e4aa036b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/9fa082f6f44248ddbcd71895e4aa036b 2024-12-10T08:01:19,844 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/9fa082f6f44248ddbcd71895e4aa036b, delay=901 ms 2024-12-10T08:01:19,845 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/9fa082f6f44248ddbcd71895e4aa036b, entries=2, sequenceid=15, filesize=5.3 K 2024-12-10T08:01:19,846 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false 2024-12-10T08:01:19,846 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T08:01:19,848 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/62fcd0cf725e4dc59434661df32df724 2024-12-10T08:01:19,849 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/0452b6421865453a9c96a5cf69151bfa 2024-12-10T08:01:19,849 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/9fa082f6f44248ddbcd71895e4aa036b 2024-12-10T08:01:19,853 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-10T08:01:19,853 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T08:01:19,854 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:19,854 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733817679761Running coprocessor pre-close hooks at 1733817679761Disabling compacts and flushes for region at 1733817679761Disabling writes for close at 1733817679762 (+1 ms)Obtaining lock to block concurrent updates at 1733817679762Preparing flush snapshotting stores in 1588230740 at 1733817679762Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733817679763 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733817679764 (+1 ms)Flushing 1588230740/info: creating writer at 1733817679764Flushing 1588230740/info: appending metadata at 1733817679791 (+27 ms)Flushing 1588230740/info: closing flushed file at 1733817679792 (+1 ms)Flushing 1588230740/ns: creating writer at 1733817679795 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733817679812 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733817679812Flushing 1588230740/table: creating writer at 1733817679815 (+3 ms)Flushing 1588230740/table: appending metadata at 1733817679834 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733817679834Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d021950: reopening flushed file at 1733817679838 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4025b46a: reopening flushed file at 1733817679840 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fddbd95: reopening flushed file at 1733817679843 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 84ms, sequenceid=15, compaction requested=false at 1733817679846 (+3 ms)Writing region close event to WAL at 1733817679849 (+3 ms)Running coprocessor post-close hooks at 1733817679853 (+4 ms)Closed at 1733817679854 (+1 ms) 2024-12-10T08:01:19,854 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:19,900 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:19,900 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46651-0x1000ecbe7810002, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:19,900 INFO [RS:1;80fbf959555f:46651 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T08:01:19,900 INFO [RS:1;80fbf959555f:46651 {}] regionserver.HRegionServer(1031): Exiting; stopping=80fbf959555f,46651,1733817675100; zookeeper connection closed. 2024-12-10T08:01:19,901 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@9f20a1a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@9f20a1a 2024-12-10T08:01:19,962 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(976): stopping server 80fbf959555f,40721,1733817674978; all regions closed. 2024-12-10T08:01:19,966 DEBUG [RS:0;80fbf959555f:40721 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:19,967 INFO [RS:0;80fbf959555f:40721 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C40721%2C1733817674978.meta:.meta(num 1733817676927) 2024-12-10T08:01:19,971 DEBUG [RS:0;80fbf959555f:40721 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:19,971 INFO [RS:0;80fbf959555f:40721 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C40721%2C1733817674978:(num 1733817676613) 2024-12-10T08:01:19,971 DEBUG [RS:0;80fbf959555f:40721 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:19,971 INFO [RS:0;80fbf959555f:40721 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:19,972 INFO [RS:0;80fbf959555f:40721 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T08:01:19,972 INFO [RS:0;80fbf959555f:40721 {}] hbase.ChoreService(370): Chore service for: regionserver/80fbf959555f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T08:01:19,972 INFO [RS:0;80fbf959555f:40721 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T08:01:19,972 INFO [regionserver/80fbf959555f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T08:01:19,973 INFO [RS:0;80fbf959555f:40721 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40721 2024-12-10T08:01:19,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80fbf959555f,40721,1733817674978 2024-12-10T08:01:19,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:19,997 INFO [RS:0;80fbf959555f:40721 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T08:01:20,007 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80fbf959555f,40721,1733817674978] 2024-12-10T08:01:20,015 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80fbf959555f,40721,1733817674978 already deleted, retry=false 2024-12-10T08:01:20,016 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80fbf959555f,40721,1733817674978 expired; onlineServers=0 2024-12-10T08:01:20,016 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '80fbf959555f,44683,1733817674202' ***** 2024-12-10T08:01:20,016 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T08:01:20,016 INFO [M:0;80fbf959555f:44683 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T08:01:20,016 INFO [M:0;80fbf959555f:44683 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T08:01:20,016 DEBUG [M:0;80fbf959555f:44683 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T08:01:20,016 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T08:01:20,016 DEBUG [M:0;80fbf959555f:44683 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T08:01:20,016 DEBUG [master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.small.0-1733817676175 {}] cleaner.HFileCleaner(306): Exit Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.small.0-1733817676175,5,FailOnTimeoutGroup] 2024-12-10T08:01:20,016 DEBUG [master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.large.0-1733817676174 {}] cleaner.HFileCleaner(306): Exit Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.large.0-1733817676174,5,FailOnTimeoutGroup] 2024-12-10T08:01:20,017 INFO [M:0;80fbf959555f:44683 {}] hbase.ChoreService(370): Chore service for: master/80fbf959555f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T08:01:20,017 INFO [M:0;80fbf959555f:44683 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T08:01:20,017 DEBUG [M:0;80fbf959555f:44683 {}] master.HMaster(1795): Stopping service threads 2024-12-10T08:01:20,017 INFO [M:0;80fbf959555f:44683 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T08:01:20,017 INFO [M:0;80fbf959555f:44683 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T08:01:20,018 INFO [M:0;80fbf959555f:44683 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T08:01:20,018 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T08:01:20,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T08:01:20,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,024 DEBUG [M:0;80fbf959555f:44683 {}] zookeeper.ZKUtil(347): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T08:01:20,024 WARN [M:0;80fbf959555f:44683 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T08:01:20,025 INFO [M:0;80fbf959555f:44683 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.lastflushedseqids 2024-12-10T08:01:20,034 INFO [M:0;80fbf959555f:44683 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T08:01:20,034 INFO [M:0;80fbf959555f:44683 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T08:01:20,035 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T08:01:20,035 INFO [M:0;80fbf959555f:44683 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,035 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,035 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T08:01:20,035 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,035 INFO [M:0;80fbf959555f:44683 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-10T08:01:20,043 DEBUG [M:0;80fbf959555f:44683 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 305a556abdce4728bbb4db57e4c4199e 2024-12-10T08:01:20,054 DEBUG [M:0;80fbf959555f:44683 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/305a556abdce4728bbb4db57e4c4199e is 82, key is hbase:meta,,1/info:regioninfo/1733817676981/Put/seqid=0 2024-12-10T08:01:20,055 INFO [M:0;80fbf959555f:44683 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/305a556abdce4728bbb4db57e4c4199e 2024-12-10T08:01:20,061 DEBUG [M:0;80fbf959555f:44683 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 14c53bb1779c4d3f91c76b5f442292bc 2024-12-10T08:01:20,072 DEBUG [M:0;80fbf959555f:44683 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14c53bb1779c4d3f91c76b5f442292bc is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733817677785/Put/seqid=0 2024-12-10T08:01:20,073 INFO [M:0;80fbf959555f:44683 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14c53bb1779c4d3f91c76b5f442292bc 2024-12-10T08:01:20,079 DEBUG [M:0;80fbf959555f:44683 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d17d0dad51854dfba0a00d248a24d267 2024-12-10T08:01:20,089 DEBUG [M:0;80fbf959555f:44683 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d17d0dad51854dfba0a00d248a24d267 is 69, key is 80fbf959555f,40721,1733817674978/rs:state/1733817676327/Put/seqid=0 2024-12-10T08:01:20,090 INFO [M:0;80fbf959555f:44683 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d17d0dad51854dfba0a00d248a24d267 2024-12-10T08:01:20,092 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/305a556abdce4728bbb4db57e4c4199e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/305a556abdce4728bbb4db57e4c4199e 2024-12-10T08:01:20,093 INFO [M:0;80fbf959555f:44683 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/305a556abdce4728bbb4db57e4c4199e, entries=8, sequenceid=84, filesize=5.5 K 2024-12-10T08:01:20,093 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/14c53bb1779c4d3f91c76b5f442292bc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/14c53bb1779c4d3f91c76b5f442292bc 2024-12-10T08:01:20,095 INFO [M:0;80fbf959555f:44683 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/14c53bb1779c4d3f91c76b5f442292bc, entries=9, sequenceid=84, filesize=6.5 K 2024-12-10T08:01:20,095 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d17d0dad51854dfba0a00d248a24d267 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d17d0dad51854dfba0a00d248a24d267 2024-12-10T08:01:20,096 INFO [M:0;80fbf959555f:44683 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d17d0dad51854dfba0a00d248a24d267, entries=2, sequenceid=84, filesize=5.1 K 2024-12-10T08:01:20,097 INFO [M:0;80fbf959555f:44683 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 62ms, sequenceid=84, compaction requested=false 2024-12-10T08:01:20,099 INFO [M:0;80fbf959555f:44683 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,099 DEBUG [M:0;80fbf959555f:44683 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733817680034Disabling compacts and flushes for region at 1733817680035 (+1 ms)Disabling writes for close at 1733817680035Obtaining lock to block concurrent updates at 1733817680035Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733817680035Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733817680035Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733817680036 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733817680036Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733817680054 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733817680054Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733817680056 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733817680072 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733817680072Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733817680074 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733817680089 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733817680089Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1733817680091 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1733817680093 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26e78092: reopening flushed file at 1733817680095 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 62ms, sequenceid=84, compaction requested=false at 1733817680097 (+2 ms)Writing region close event to WAL at 1733817680099 (+2 ms)Closed at 1733817680099 2024-12-10T08:01:20,101 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T08:01:20,101 INFO [M:0;80fbf959555f:44683 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T08:01:20,101 INFO [M:0;80fbf959555f:44683 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44683 2024-12-10T08:01:20,101 INFO [M:0;80fbf959555f:44683 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T08:01:20,107 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:20,107 INFO [RS:0;80fbf959555f:40721 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T08:01:20,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40721-0x1000ecbe7810001, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:20,108 INFO [RS:0;80fbf959555f:40721 {}] regionserver.HRegionServer(1031): Exiting; stopping=80fbf959555f,40721,1733817674978; zookeeper connection closed. 2024-12-10T08:01:20,108 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@711d7dd5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@711d7dd5 2024-12-10T08:01:20,108 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-10T08:01:20,224 INFO [M:0;80fbf959555f:44683 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T08:01:20,224 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:20,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44683-0x1000ecbe7810000, quorum=127.0.0.1:51160, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:20,232 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T08:01:20,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T08:01:20,250 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,257 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/80fbf959555f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1733817678125 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/80fbf959555f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733817679841 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/80fbf959555f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733817679672 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/80fbf959555f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1733817679839 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=119 (was 129), ProcessCount=11 (was 11), AvailableMemoryMB=7109 (was 7471) 2024-12-10T08:01:20,262 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=119, ProcessCount=11, AvailableMemoryMB=7109 2024-12-10T08:01:20,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/hadoop-log-dir so I do NOT create it in target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3 2024-12-10T08:01:20,262 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/hadoop-tmp-dir so I do NOT create it in target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3 2024-12-10T08:01:20,262 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,265 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/cluster_e03ecf9c-09e5-1a4a-c955-90487a91ffa1, deleteOnExit=true 2024-12-10T08:01:20,268 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/cluster_e03ecf9c-09e5-1a4a-c955-90487a91ffa1/zookeeper_0, clientPort=57631, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/cluster_e03ecf9c-09e5-1a4a-c955-90487a91ffa1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/cluster_e03ecf9c-09e5-1a4a-c955-90487a91ffa1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T08:01:20,269 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57631 2024-12-10T08:01:20,280 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 with version=8 2024-12-10T08:01:20,280 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/hbase-staging 2024-12-10T08:01:20,283 INFO [Time-limited test {}] client.ConnectionUtils(128): master/80fbf959555f:0 server-side Connection retries=45 2024-12-10T08:01:20,283 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,284 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T08:01:20,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,284 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T08:01:20,284 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T08:01:20,284 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T08:01:20,285 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35517 2024-12-10T08:01:20,286 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35517 connecting to ZooKeeper ensemble=127.0.0.1:57631 2024-12-10T08:01:20,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:355170x0, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T08:01:20,328 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35517-0x1000ecc00270000 connected 2024-12-10T08:01:20,397 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,397 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,400 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:20,400 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4, hbase.cluster.distributed=false 2024-12-10T08:01:20,405 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T08:01:20,406 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35517 2024-12-10T08:01:20,407 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35517 2024-12-10T08:01:20,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35517 2024-12-10T08:01:20,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35517 2024-12-10T08:01:20,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35517 2024-12-10T08:01:20,422 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80fbf959555f:0 server-side Connection retries=45 2024-12-10T08:01:20,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,423 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T08:01:20,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T08:01:20,423 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T08:01:20,423 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T08:01:20,424 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39231 2024-12-10T08:01:20,425 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39231 connecting to ZooKeeper ensemble=127.0.0.1:57631 2024-12-10T08:01:20,425 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,425 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:392310x0, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T08:01:20,438 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39231-0x1000ecc00270001 connected 2024-12-10T08:01:20,438 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:20,439 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T08:01:20,440 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.cache 2024-12-10T08:01:20,440 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T08:01:20,440 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T08:01:20,441 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T08:01:20,441 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T08:01:20,441 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T08:01:20,441 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T08:01:20,442 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T08:01:20,442 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence is complete. 2024-12-10T08:01:20,442 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T08:01:20,443 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T08:01:20,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T08:01:20,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39231 2024-12-10T08:01:20,446 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39231 2024-12-10T08:01:20,447 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39231 2024-12-10T08:01:20,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39231 2024-12-10T08:01:20,448 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39231 2024-12-10T08:01:20,461 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80fbf959555f:0 server-side Connection retries=45 2024-12-10T08:01:20,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,461 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T08:01:20,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:20,461 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T08:01:20,462 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T08:01:20,462 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T08:01:20,462 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46433 2024-12-10T08:01:20,463 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46433 connecting to ZooKeeper ensemble=127.0.0.1:57631 2024-12-10T08:01:20,464 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,464 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464330x0, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T08:01:20,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:20,472 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46433-0x1000ecc00270002 connected 2024-12-10T08:01:20,472 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T08:01:20,473 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.cache 2024-12-10T08:01:20,473 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T08:01:20,473 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T08:01:20,473 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T08:01:20,473 INFO [Thread-122 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T08:01:20,473 WARN [Thread-122 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T08:01:20,474 INFO [Thread-122 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T08:01:20,474 INFO [Thread-122 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence is complete. 2024-12-10T08:01:20,474 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T08:01:20,474 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T08:01:20,475 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T08:01:20,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T08:01:20,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46433 2024-12-10T08:01:20,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46433 2024-12-10T08:01:20,479 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46433 2024-12-10T08:01:20,480 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46433 2024-12-10T08:01:20,481 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46433 2024-12-10T08:01:20,497 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;80fbf959555f:35517 2024-12-10T08:01:20,497 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/80fbf959555f,35517,1733817680282 2024-12-10T08:01:20,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:20,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:20,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:20,505 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/80fbf959555f,35517,1733817680282 2024-12-10T08:01:20,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T08:01:20,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T08:01:20,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,513 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,513 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T08:01:20,514 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/80fbf959555f,35517,1733817680282 from backup master directory 2024-12-10T08:01:20,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:20,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:20,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/80fbf959555f,35517,1733817680282 2024-12-10T08:01:20,521 WARN [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T08:01:20,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T08:01:20,521 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=80fbf959555f,35517,1733817680282 2024-12-10T08:01:20,522 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/hbase.id] with ID: 04f5223b-7458-447f-92f5-e425d900141e 2024-12-10T08:01:20,522 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.tmp/hbase.id 2024-12-10T08:01:20,525 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/hbase.id] 2024-12-10T08:01:20,558 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:20,558 INFO [master/80fbf959555f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T08:01:20,561 INFO [master/80fbf959555f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-12-10T08:01:20,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,571 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,578 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T08:01:20,579 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T08:01:20,579 INFO [master/80fbf959555f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:20,597 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store 2024-12-10T08:01:20,604 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:20,604 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T08:01:20,604 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,604 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,604 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T08:01:20,604 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,605 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:20,605 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733817680604Disabling compacts and flushes for region at 1733817680604Disabling writes for close at 1733817680604Writing region close event to WAL at 1733817680604Closed at 1733817680605 (+1 ms) 2024-12-10T08:01:20,605 WARN [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/.initializing 2024-12-10T08:01:20,605 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/WALs/80fbf959555f,35517,1733817680282 2024-12-10T08:01:20,612 INFO [master/80fbf959555f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T08:01:20,616 INFO [master/80fbf959555f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=80fbf959555f%2C35517%2C1733817680282, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/WALs/80fbf959555f,35517,1733817680282, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/oldWALs, maxLogs=10 2024-12-10T08:01:20,630 INFO [master/80fbf959555f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/WALs/80fbf959555f,35517,1733817680282/80fbf959555f%2C35517%2C1733817680282.1733817680616 2024-12-10T08:01:20,630 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:20,630 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:20,630 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:20,630 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,631 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,631 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,635 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T08:01:20,635 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:20,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:20,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T08:01:20,639 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:20,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:20,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,642 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T08:01:20,642 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:20,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:20,643 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T08:01:20,646 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:20,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:20,646 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,647 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,647 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,647 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,647 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,647 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T08:01:20,648 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T08:01:20,653 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T08:01:20,654 INFO [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74153269, jitterRate=0.1049698144197464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T08:01:20,655 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733817680631Initializing all the Stores at 1733817680631Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817680631Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817680631Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817680631Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817680631Cleaning up temporary data from old regions at 1733817680647 (+16 ms)Region opened successfully at 1733817680654 (+7 ms) 2024-12-10T08:01:20,655 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T08:01:20,660 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1efa481c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80fbf959555f/172.17.0.2:0 2024-12-10T08:01:20,661 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T08:01:20,661 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T08:01:20,661 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T08:01:20,662 INFO [master/80fbf959555f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T08:01:20,662 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-10T08:01:20,662 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-10T08:01:20,662 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T08:01:20,665 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T08:01:20,666 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T08:01:20,757 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T08:01:20,758 INFO [master/80fbf959555f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T08:01:20,761 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T08:01:20,871 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T08:01:20,872 INFO [master/80fbf959555f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T08:01:20,877 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T08:01:20,887 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T08:01:20,889 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T08:01:20,896 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T08:01:20,900 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T08:01:20,911 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T08:01:20,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:20,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:20,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:20,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,923 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=80fbf959555f,35517,1733817680282, sessionid=0x1000ecc00270000, setting cluster-up flag (Was=false) 2024-12-10T08:01:20,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,979 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T08:01:20,981 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80fbf959555f,35517,1733817680282 2024-12-10T08:01:20,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:20,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:21,021 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T08:01:21,022 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=80fbf959555f,35517,1733817680282 2024-12-10T08:01:21,023 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T08:01:21,024 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:21,024 INFO [master/80fbf959555f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T08:01:21,025 INFO [master/80fbf959555f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T08:01:21,025 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 80fbf959555f,35517,1733817680282 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T08:01:21,027 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:21,027 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:21,027 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:21,027 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/80fbf959555f:0, corePoolSize=5, maxPoolSize=5 2024-12-10T08:01:21,027 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/80fbf959555f:0, corePoolSize=10, maxPoolSize=10 2024-12-10T08:01:21,027 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,027 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/80fbf959555f:0, corePoolSize=2, maxPoolSize=2 2024-12-10T08:01:21,028 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,029 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:21,029 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T08:01:21,030 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,030 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T08:01:21,031 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733817711031 2024-12-10T08:01:21,031 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T08:01:21,031 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T08:01:21,031 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T08:01:21,031 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T08:01:21,031 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T08:01:21,031 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T08:01:21,032 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,032 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T08:01:21,032 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T08:01:21,033 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T08:01:21,033 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T08:01:21,033 INFO [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T08:01:21,034 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.large.0-1733817681033,5,FailOnTimeoutGroup] 2024-12-10T08:01:21,035 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.small.0-1733817681034,5,FailOnTimeoutGroup] 2024-12-10T08:01:21,035 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,035 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T08:01:21,035 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,035 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,047 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T08:01:21,047 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:21,052 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:21,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T08:01:21,056 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T08:01:21,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T08:01:21,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T08:01:21,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T08:01:21,061 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T08:01:21,061 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,062 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T08:01:21,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T08:01:21,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,065 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T08:01:21,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:21,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:21,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T08:01:21,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T08:01:21,066 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T08:01:21,066 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T08:01:21,072 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T08:01:21,072 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67310968, jitterRate=0.0030115842819213867}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T08:01:21,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733817681052Initializing all the Stores at 1733817681052Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817681052Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817681053 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817681053Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817681053Cleaning up temporary data from old regions at 1733817681066 (+13 ms)Region opened successfully at 1733817681073 (+7 ms) 2024-12-10T08:01:21,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T08:01:21,073 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T08:01:21,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T08:01:21,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T08:01:21,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T08:01:21,073 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:21,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733817681073Disabling compacts and flushes for region at 1733817681073Disabling writes for close at 1733817681073Writing region close event to WAL at 1733817681073Closed at 1733817681073 2024-12-10T08:01:21,074 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:21,074 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T08:01:21,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T08:01:21,075 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T08:01:21,076 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T08:01:21,084 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(746): ClusterId : 04f5223b-7458-447f-92f5-e425d900141e 2024-12-10T08:01:21,084 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(746): ClusterId : 04f5223b-7458-447f-92f5-e425d900141e 2024-12-10T08:01:21,084 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T08:01:21,084 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T08:01:21,097 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T08:01:21,097 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T08:01:21,097 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T08:01:21,097 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T08:01:21,106 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T08:01:21,106 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T08:01:21,107 DEBUG [RS:0;80fbf959555f:39231 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b79e4b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80fbf959555f/172.17.0.2:0 2024-12-10T08:01:21,107 DEBUG [RS:1;80fbf959555f:46433 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2393c670, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80fbf959555f/172.17.0.2:0 2024-12-10T08:01:21,120 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;80fbf959555f:46433 2024-12-10T08:01:21,121 INFO [RS:1;80fbf959555f:46433 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T08:01:21,121 INFO [RS:1;80fbf959555f:46433 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T08:01:21,121 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T08:01:21,121 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;80fbf959555f:39231 2024-12-10T08:01:21,122 INFO [RS:0;80fbf959555f:39231 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T08:01:21,122 INFO [RS:0;80fbf959555f:39231 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T08:01:21,122 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T08:01:21,122 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(2659): reportForDuty to master=80fbf959555f,35517,1733817680282 with port=46433, startcode=1733817680461 2024-12-10T08:01:21,122 DEBUG [RS:1;80fbf959555f:46433 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T08:01:21,122 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(2659): reportForDuty to master=80fbf959555f,35517,1733817680282 with port=39231, startcode=1733817680422 2024-12-10T08:01:21,122 DEBUG [RS:0;80fbf959555f:39231 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T08:01:21,124 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53129, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T08:01:21,124 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59917, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T08:01:21,125 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35517 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80fbf959555f,46433,1733817680461 2024-12-10T08:01:21,125 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35517 {}] master.ServerManager(517): Registering regionserver=80fbf959555f,46433,1733817680461 2024-12-10T08:01:21,126 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35517 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,126 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35517 {}] master.ServerManager(517): Registering regionserver=80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,127 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:21,127 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T08:01:21,127 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T08:01:21,128 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:21,128 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T08:01:21,128 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T08:01:21,137 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:21,171 DEBUG [RS:1;80fbf959555f:46433 {}] zookeeper.ZKUtil(111): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80fbf959555f,46433,1733817680461 2024-12-10T08:01:21,171 WARN [RS:1;80fbf959555f:46433 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T08:01:21,171 INFO [RS:1;80fbf959555f:46433 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:21,171 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80fbf959555f,39231,1733817680422] 2024-12-10T08:01:21,172 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80fbf959555f,46433,1733817680461] 2024-12-10T08:01:21,172 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46433,1733817680461 2024-12-10T08:01:21,172 DEBUG [RS:0;80fbf959555f:39231 {}] zookeeper.ZKUtil(111): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,172 WARN [RS:0;80fbf959555f:39231 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T08:01:21,172 INFO [RS:0;80fbf959555f:39231 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:21,173 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,181 INFO [RS:1;80fbf959555f:46433 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T08:01:21,184 INFO [RS:0;80fbf959555f:39231 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T08:01:21,184 INFO [RS:1;80fbf959555f:46433 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T08:01:21,185 INFO [RS:1;80fbf959555f:46433 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T08:01:21,185 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,185 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T08:01:21,186 INFO [RS:1;80fbf959555f:46433 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T08:01:21,186 INFO [RS:0;80fbf959555f:39231 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T08:01:21,186 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,187 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,187 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,187 INFO [RS:0;80fbf959555f:39231 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T08:01:21,187 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,187 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,187 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,187 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T08:01:21,187 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,187 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80fbf959555f:0, corePoolSize=2, maxPoolSize=2 2024-12-10T08:01:21,188 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 INFO [RS:0;80fbf959555f:39231 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T08:01:21,188 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,188 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,188 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:21,189 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80fbf959555f:0, corePoolSize=2, maxPoolSize=2 2024-12-10T08:01:21,189 DEBUG [RS:1;80fbf959555f:46433 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:21,189 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,189 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,189 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,189 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,189 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,189 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,189 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,189 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,189 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,189 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,189 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,46433,1733817680461-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T08:01:21,190 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:21,190 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:21,190 DEBUG [RS:0;80fbf959555f:39231 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:21,190 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,190 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,190 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,190 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,190 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,191 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,39231,1733817680422-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T08:01:21,204 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T08:01:21,204 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,39231,1733817680422-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,205 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,205 INFO [RS:0;80fbf959555f:39231 {}] regionserver.Replication(171): 80fbf959555f,39231,1733817680422 started 2024-12-10T08:01:21,209 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T08:01:21,209 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,46433,1733817680461-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,210 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,210 INFO [RS:1;80fbf959555f:46433 {}] regionserver.Replication(171): 80fbf959555f,46433,1733817680461 started 2024-12-10T08:01:21,216 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,216 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1482): Serving as 80fbf959555f,39231,1733817680422, RpcServer on 80fbf959555f/172.17.0.2:39231, sessionid=0x1000ecc00270001 2024-12-10T08:01:21,217 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T08:01:21,217 DEBUG [RS:0;80fbf959555f:39231 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,217 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,39231,1733817680422' 2024-12-10T08:01:21,217 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T08:01:21,218 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T08:01:21,218 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T08:01:21,218 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T08:01:21,218 DEBUG [RS:0;80fbf959555f:39231 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,218 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,39231,1733817680422' 2024-12-10T08:01:21,218 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T08:01:21,219 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T08:01:21,219 DEBUG [RS:0;80fbf959555f:39231 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T08:01:21,219 INFO [RS:0;80fbf959555f:39231 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T08:01:21,220 INFO [RS:0;80fbf959555f:39231 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T08:01:21,226 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,226 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1482): Serving as 80fbf959555f,46433,1733817680461, RpcServer on 80fbf959555f/172.17.0.2:46433, sessionid=0x1000ecc00270002 2024-12-10T08:01:21,226 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T08:01:21,226 DEBUG [RS:1;80fbf959555f:46433 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80fbf959555f,46433,1733817680461 2024-12-10T08:01:21,226 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,46433,1733817680461' 2024-12-10T08:01:21,226 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T08:01:21,226 WARN [80fbf959555f:35517 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T08:01:21,227 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T08:01:21,227 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T08:01:21,227 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T08:01:21,227 DEBUG [RS:1;80fbf959555f:46433 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80fbf959555f,46433,1733817680461 2024-12-10T08:01:21,227 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,46433,1733817680461' 2024-12-10T08:01:21,227 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T08:01:21,228 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T08:01:21,228 DEBUG [RS:1;80fbf959555f:46433 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T08:01:21,228 INFO [RS:1;80fbf959555f:46433 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T08:01:21,228 INFO [RS:1;80fbf959555f:46433 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T08:01:21,321 INFO [RS:0;80fbf959555f:39231 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T08:01:21,323 INFO [RS:0;80fbf959555f:39231 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C39231%2C1733817680422, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:21,329 INFO [RS:1;80fbf959555f:46433 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T08:01:21,330 INFO [RS:1;80fbf959555f:46433 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C46433%2C1733817680461, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46433,1733817680461, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:21,343 INFO [RS:0;80fbf959555f:39231 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422/80fbf959555f%2C39231%2C1733817680422.1733817681326 2024-12-10T08:01:21,344 DEBUG [RS:0;80fbf959555f:39231 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:21,348 INFO [RS:1;80fbf959555f:46433 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46433,1733817680461/80fbf959555f%2C46433%2C1733817680461.1733817681331 2024-12-10T08:01:21,348 DEBUG [RS:1;80fbf959555f:46433 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:21,477 DEBUG [80fbf959555f:35517 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T08:01:21,477 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(204): Hosts are {80fbf959555f=0} racks are {/default-rack=0} 2024-12-10T08:01:21,479 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T08:01:21,479 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T08:01:21,479 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T08:01:21,479 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T08:01:21,479 INFO [80fbf959555f:35517 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T08:01:21,479 INFO [80fbf959555f:35517 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T08:01:21,480 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T08:01:21,480 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,482 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80fbf959555f,39231,1733817680422, state=OPENING 2024-12-10T08:01:21,546 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T08:01:21,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:21,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:21,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:21,556 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T08:01:21,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:21,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:21,556 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=80fbf959555f,39231,1733817680422}] 2024-12-10T08:01:21,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:21,712 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T08:01:21,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59953, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T08:01:21,720 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T08:01:21,720 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:21,720 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T08:01:21,721 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C39231%2C1733817680422.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:21,739 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422/80fbf959555f%2C39231%2C1733817680422.meta.1733817681722.meta 2024-12-10T08:01:21,740 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:21,740 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:21,740 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T08:01:21,740 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T08:01:21,741 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T08:01:21,741 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T08:01:21,741 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:21,741 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T08:01:21,741 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T08:01:21,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T08:01:21,742 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T08:01:21,742 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,743 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T08:01:21,743 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T08:01:21,743 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T08:01:21,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T08:01:21,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,744 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T08:01:21,745 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T08:01:21,745 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:21,745 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T08:01:21,746 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:21,746 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:21,747 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T08:01:21,747 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T08:01:21,747 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T08:01:21,747 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T08:01:21,748 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61179485, jitterRate=-0.08835463225841522}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T08:01:21,748 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T08:01:21,749 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733817681741Writing region info on filesystem at 1733817681741Initializing all the Stores at 1733817681741Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817681741Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817681742 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817681742Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817681742Cleaning up temporary data from old regions at 1733817681747 (+5 ms)Running coprocessor post-open hooks at 1733817681748 (+1 ms)Region opened successfully at 1733817681749 (+1 ms) 2024-12-10T08:01:21,749 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733817681711 2024-12-10T08:01:21,752 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T08:01:21,752 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T08:01:21,752 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,753 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80fbf959555f,39231,1733817680422, state=OPEN 2024-12-10T08:01:21,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:21,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:21,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:21,771 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=80fbf959555f,39231,1733817680422 2024-12-10T08:01:21,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:21,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:21,771 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:21,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T08:01:21,774 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=80fbf959555f,39231,1733817680422 in 215 msec 2024-12-10T08:01:21,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T08:01:21,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 700 msec 2024-12-10T08:01:21,778 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T08:01:21,778 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T08:01:21,782 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T08:01:21,783 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80fbf959555f,39231,1733817680422, seqNum=-1] 2024-12-10T08:01:21,783 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T08:01:21,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58783, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T08:01:21,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 767 msec 2024-12-10T08:01:21,793 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733817681793, completionTime=-1 2024-12-10T08:01:21,793 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T08:01:21,793 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T08:01:21,796 INFO [master/80fbf959555f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T08:01:21,796 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733817741796 2024-12-10T08:01:21,796 INFO [master/80fbf959555f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733817801796 2024-12-10T08:01:21,796 INFO [master/80fbf959555f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-10T08:01:21,796 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T08:01:21,797 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,35517,1733817680282-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,797 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,35517,1733817680282-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,797 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,35517,1733817680282-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,797 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-80fbf959555f:35517, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,797 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,797 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,800 DEBUG [master/80fbf959555f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T08:01:21,802 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.281sec 2024-12-10T08:01:21,802 INFO [master/80fbf959555f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T08:01:21,802 INFO [master/80fbf959555f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T08:01:21,802 INFO [master/80fbf959555f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T08:01:21,802 INFO [master/80fbf959555f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T08:01:21,803 INFO [master/80fbf959555f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T08:01:21,803 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,35517,1733817680282-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T08:01:21,803 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,35517,1733817680282-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T08:01:21,803 DEBUG [master/80fbf959555f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T08:01:21,803 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T08:01:21,803 INFO [master/80fbf959555f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,35517,1733817680282-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:21,885 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d25b74f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T08:01:21,885 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 80fbf959555f,35517,-1 for getting cluster id 2024-12-10T08:01:21,885 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T08:01:21,889 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '04f5223b-7458-447f-92f5-e425d900141e' 2024-12-10T08:01:21,891 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T08:01:21,891 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "04f5223b-7458-447f-92f5-e425d900141e" 2024-12-10T08:01:21,892 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197b6262, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T08:01:21,892 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [80fbf959555f,35517,-1] 2024-12-10T08:01:21,893 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T08:01:21,893 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:21,895 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T08:01:21,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5858d96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T08:01:21,897 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T08:01:21,898 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80fbf959555f,39231,1733817680422, seqNum=-1] 2024-12-10T08:01:21,898 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T08:01:21,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T08:01:21,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=80fbf959555f,35517,1733817680282 2024-12-10T08:01:21,903 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T08:01:21,904 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 80fbf959555f,35517,1733817680282 2024-12-10T08:01:21,904 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1a906cfb 2024-12-10T08:01:21,904 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T08:01:21,906 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T08:01:21,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35517 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733817681903', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T08:01:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35517 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 2024-12-10T08:01:21,909 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T08:01:21,909 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:21,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35517 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733817681903" procId is: 4 2024-12-10T08:01:21,910 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T08:01:21,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35517 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:21,918 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fd745abd220d6cf27defd98bdba14e28, NAME => 'testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733817681903', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:21,924 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:21,924 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(1722): Closing fd745abd220d6cf27defd98bdba14e28, disabling compactions & flushes 2024-12-10T08:01:21,924 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:21,924 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:21,924 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. after waiting 0 ms 2024-12-10T08:01:21,924 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:21,924 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:21,924 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733817681903-pool-0 {}] regionserver.HRegion(1676): Region close journal for fd745abd220d6cf27defd98bdba14e28: Waiting for close lock at 1733817681924Disabling compacts and flushes for region at 1733817681924Disabling writes for close at 1733817681924Writing region close event to WAL at 1733817681924Closed at 1733817681924 2024-12-10T08:01:21,925 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T08:01:21,925 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733817681925"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733817681925"}]},"ts":"1733817681925"} 2024-12-10T08:01:21,928 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T08:01:21,929 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T08:01:21,929 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733817681903","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733817681929"}]},"ts":"1733817681929"} 2024-12-10T08:01:21,931 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733817681903, state=ENABLING in hbase:meta 2024-12-10T08:01:21,931 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {80fbf959555f=0} racks are {/default-rack=0} 2024-12-10T08:01:21,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T08:01:21,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T08:01:21,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T08:01:21,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T08:01:21,932 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T08:01:21,932 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T08:01:21,932 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T08:01:21,932 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN}] 2024-12-10T08:01:21,933 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN 2024-12-10T08:01:21,934 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN; state=OFFLINE, location=80fbf959555f,39231,1733817680422; forceNewPlan=false, retain=false 2024-12-10T08:01:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35517 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:22,084 INFO [80fbf959555f:35517 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T08:01:22,085 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd745abd220d6cf27defd98bdba14e28, regionState=OPENING, regionLocation=80fbf959555f,39231,1733817680422 2024-12-10T08:01:22,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN because future has completed 2024-12-10T08:01:22,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd745abd220d6cf27defd98bdba14e28, server=80fbf959555f,39231,1733817680422}] 2024-12-10T08:01:22,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35517 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:22,244 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:22,244 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fd745abd220d6cf27defd98bdba14e28, NAME => 'testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28.', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:22,244 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733817681903 fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,244 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:22,244 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,245 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,245 INFO [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,248 INFO [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd745abd220d6cf27defd98bdba14e28 columnFamilyName family 2024-12-10T08:01:22,248 DEBUG [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:22,248 INFO [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] regionserver.HStore(327): Store=fd745abd220d6cf27defd98bdba14e28/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:22,248 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,249 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,249 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,249 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,249 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,250 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,255 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T08:01:22,255 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fd745abd220d6cf27defd98bdba14e28; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72107430, jitterRate=0.0744844377040863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T08:01:22,255 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:22,256 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fd745abd220d6cf27defd98bdba14e28: Running coprocessor pre-open hook at 1733817682245Writing region info on filesystem at 1733817682245Initializing all the Stores at 1733817682245Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817682245Cleaning up temporary data from old regions at 1733817682249 (+4 ms)Running coprocessor post-open hooks at 1733817682255 (+6 ms)Region opened successfully at 1733817682256 (+1 ms) 2024-12-10T08:01:22,256 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28., pid=6, masterSystemTime=1733817682240 2024-12-10T08:01:22,259 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:22,259 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:22,259 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fd745abd220d6cf27defd98bdba14e28, regionState=OPEN, openSeqNum=2, regionLocation=80fbf959555f,39231,1733817680422 2024-12-10T08:01:22,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd745abd220d6cf27defd98bdba14e28, server=80fbf959555f,39231,1733817680422 because future has completed 2024-12-10T08:01:22,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T08:01:22,264 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fd745abd220d6cf27defd98bdba14e28, server=80fbf959555f,39231,1733817680422 in 174 msec 2024-12-10T08:01:22,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T08:01:22,266 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN in 332 msec 2024-12-10T08:01:22,266 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T08:01:22,266 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733817681903","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733817682266"}]},"ts":"1733817682266"} 2024-12-10T08:01:22,268 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733817681903, state=ENABLED in hbase:meta 2024-12-10T08:01:22,269 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T08:01:22,270 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733817681903 in 362 msec 2024-12-10T08:01:22,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T08:01:22,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35517 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T08:01:22,537 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733817681903 completed 2024-12-10T08:01:22,537 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733817681903 get assigned. Timeout = 60000ms 2024-12-10T08:01:22,537 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T08:01:22,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733817681903 assigned to meta. Checking AM states. 2024-12-10T08:01:22,541 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T08:01:22,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733817681903 assigned. 2024-12-10T08:01:22,547 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733817681903', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28., hostname=80fbf959555f,39231,1733817680422, seqNum=2] 2024-12-10T08:01:22,552 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fd745abd220d6cf27defd98bdba14e28 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T08:01:22,561 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f3d8cb50d1c544d58e014ead3f9c3aac 2024-12-10T08:01:22,574 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/.tmp/family/f3d8cb50d1c544d58e014ead3f9c3aac is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T08:01:22,575 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/.tmp/family/f3d8cb50d1c544d58e014ead3f9c3aac 2024-12-10T08:01:22,577 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/.tmp/family/f3d8cb50d1c544d58e014ead3f9c3aac as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac 2024-12-10T08:01:22,578 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac, delay=936 ms 2024-12-10T08:01:22,579 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T08:01:22,580 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for fd745abd220d6cf27defd98bdba14e28 in 28ms, sequenceid=6, compaction requested=false 2024-12-10T08:01:22,580 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733817681903' 2024-12-10T08:01:22,580 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fd745abd220d6cf27defd98bdba14e28: 2024-12-10T08:01:23,515 DEBUG [hfile-prefetch-1733817679672 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f3d8cb50d1c544d58e014ead3f9c3aac_0. 2024-12-10T08:01:23,516 DEBUG [hfile-prefetch-1733817679672 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f3d8cb50d1c544d58e014ead3f9c3aac_117. 2024-12-10T08:01:23,516 DEBUG [hfile-prefetch-1733817679672 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac, with total blocks 2, and data blocks 1 2024-12-10T08:01:23,517 DEBUG [hfile-prefetch-1733817679672 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T08:01:23,517 DEBUG [hfile-prefetch-1733817679672 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T08:01:23,517 DEBUG [hfile-prefetch-1733817679672 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac. Will sleep 100ms and try the verification again. 2024-12-10T08:01:23,581 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;80fbf959555f:39231,5,FailOnTimeoutGroup] 2024-12-10T08:01:23,581 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80fbf959555f,39231,1733817680422' ***** 2024-12-10T08:01:23,581 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-10T08:01:23,581 INFO [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T08:01:23,591 INFO [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence 2024-12-10T08:01:23,591 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T08:01:23,591 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T08:01:23,591 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T08:01:23,591 INFO [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T08:01:23,591 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T08:01:23,593 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T08:01:23,594 INFO [regionserver/80fbf959555f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:23,602 DEBUG [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1487): Thread RS:0;80fbf959555f:39231 started persisting bucket cache to file 2024-12-10T08:01:23,602 DEBUG [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:23,602 DEBUG [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:23,604 DEBUG [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:23,604 DEBUG [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817683602 2024-12-10T08:01:23,604 DEBUG [RS:0;80fbf959555f:39231 {}] bucket.BucketCache(1509): Thread RS:0;80fbf959555f:39231 finished persisting bucket cache to file, renaming 2024-12-10T08:01:23,604 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T08:01:23,604 INFO [RS:0;80fbf959555f:39231 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(3091): Received CLOSE for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(959): stopping server 80fbf959555f,39231,1733817680422 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;80fbf959555f:39231. 2024-12-10T08:01:23,605 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T08:01:23,605 DEBUG [RS:0;80fbf959555f:39231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:23,605 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd745abd220d6cf27defd98bdba14e28, disabling compactions & flushes 2024-12-10T08:01:23,605 DEBUG [RS:0;80fbf959555f:39231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:23,605 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:23,605 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T08:01:23,605 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. after waiting 0 ms 2024-12-10T08:01:23,605 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T08:01:23,605 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T08:01:23,606 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1325): Online Regions={fd745abd220d6cf27defd98bdba14e28=testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28., 1588230740=hbase:meta,,1.1588230740} 2024-12-10T08:01:23,606 DEBUG [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:23,606 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T08:01:23,606 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T08:01:23,606 DEBUG [StoreFileCloser-fd745abd220d6cf27defd98bdba14e28-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac 2024-12-10T08:01:23,606 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T08:01:23,606 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T08:01:23,606 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T08:01:23,606 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T08:01:23,606 WARN [hfile-prefetch-1733817679672 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-10T08:01:23,606 DEBUG [hfile-prefetch-1733817679672 {}] hfile.PrefetchExecutor(121): Prefetch completed for f3d8cb50d1c544d58e014ead3f9c3aac 2024-12-10T08:01:23,606 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-10T08:01:23,610 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T08:01:23,610 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:23,610 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd745abd220d6cf27defd98bdba14e28: Waiting for close lock at 1733817683605Running coprocessor pre-close hooks at 1733817683605Disabling compacts and flushes for region at 1733817683605Disabling writes for close at 1733817683605Writing region close event to WAL at 1733817683606 (+1 ms)Running coprocessor post-close hooks at 1733817683610 (+4 ms)Closed at 1733817683610 2024-12-10T08:01:23,610 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:23,614 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on db79825b0e444d04adfd1c229266667b 2024-12-10T08:01:23,627 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/db79825b0e444d04adfd1c229266667b is 209, key is testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28./info:regioninfo/1733817682259/Put/seqid=0 2024-12-10T08:01:23,628 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/db79825b0e444d04adfd1c229266667b 2024-12-10T08:01:23,636 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5fd43728d70e45e094bb7fc594fb8896 2024-12-10T08:01:23,646 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/ns/5fd43728d70e45e094bb7fc594fb8896 is 43, key is default/ns:d/1733817681786/Put/seqid=0 2024-12-10T08:01:23,647 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/ns/5fd43728d70e45e094bb7fc594fb8896 2024-12-10T08:01:23,654 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 90333b1822ef473b910cd8a1db856df4 2024-12-10T08:01:23,668 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/table/90333b1822ef473b910cd8a1db856df4 is 80, key is testBlockEvictionOnGracefulStop1733817681903/table:state/1733817682266/Put/seqid=0 2024-12-10T08:01:23,669 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/table/90333b1822ef473b910cd8a1db856df4 2024-12-10T08:01:23,672 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/db79825b0e444d04adfd1c229266667b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/db79825b0e444d04adfd1c229266667b 2024-12-10T08:01:23,673 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/db79825b0e444d04adfd1c229266667b, delay=988 ms 2024-12-10T08:01:23,674 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/db79825b0e444d04adfd1c229266667b, entries=10, sequenceid=11, filesize=7.0 K 2024-12-10T08:01:23,675 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/ns/5fd43728d70e45e094bb7fc594fb8896 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/5fd43728d70e45e094bb7fc594fb8896 2024-12-10T08:01:23,676 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/5fd43728d70e45e094bb7fc594fb8896, delay=935 ms 2024-12-10T08:01:23,676 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T08:01:23,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T08:01:23,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:23,676 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:23,677 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/5fd43728d70e45e094bb7fc594fb8896, entries=2, sequenceid=11, filesize=5.0 K 2024-12-10T08:01:23,678 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/table/90333b1822ef473b910cd8a1db856df4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/90333b1822ef473b910cd8a1db856df4 2024-12-10T08:01:23,679 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:23,679 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817683676 2024-12-10T08:01:23,679 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T08:01:23,679 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/90333b1822ef473b910cd8a1db856df4, delay=940 ms 2024-12-10T08:01:23,680 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/90333b1822ef473b910cd8a1db856df4, entries=2, sequenceid=11, filesize=5.3 K 2024-12-10T08:01:23,681 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=11, compaction requested=false 2024-12-10T08:01:23,681 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/db79825b0e444d04adfd1c229266667b 2024-12-10T08:01:23,682 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/5fd43728d70e45e094bb7fc594fb8896 2024-12-10T08:01:23,683 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/90333b1822ef473b910cd8a1db856df4 2024-12-10T08:01:23,686 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-10T08:01:23,686 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T08:01:23,686 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:23,687 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733817683606Running coprocessor pre-close hooks at 1733817683606Disabling compacts and flushes for region at 1733817683606Disabling writes for close at 1733817683606Obtaining lock to block concurrent updates at 1733817683606Preparing flush snapshotting stores in 1588230740 at 1733817683606Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733817683607 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733817683607Flushing 1588230740/info: creating writer at 1733817683607Flushing 1588230740/info: appending metadata at 1733817683626 (+19 ms)Flushing 1588230740/info: closing flushed file at 1733817683627 (+1 ms)Flushing 1588230740/ns: creating writer at 1733817683630 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733817683645 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733817683645Flushing 1588230740/table: creating writer at 1733817683649 (+4 ms)Flushing 1588230740/table: appending metadata at 1733817683668 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733817683668Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21c3edda: reopening flushed file at 1733817683671 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@333d519c: reopening flushed file at 1733817683675 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21c68186: reopening flushed file at 1733817683678 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=11, compaction requested=false at 1733817683681 (+3 ms)Writing region close event to WAL at 1733817683683 (+2 ms)Running coprocessor post-close hooks at 1733817683686 (+3 ms)Closed at 1733817683686 2024-12-10T08:01:23,687 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:23,806 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(976): stopping server 80fbf959555f,39231,1733817680422; all regions closed. 2024-12-10T08:01:23,810 DEBUG [RS:0;80fbf959555f:39231 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:23,810 INFO [RS:0;80fbf959555f:39231 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C39231%2C1733817680422.meta:.meta(num 1733817681722) 2024-12-10T08:01:23,813 DEBUG [RS:0;80fbf959555f:39231 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:23,813 INFO [RS:0;80fbf959555f:39231 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C39231%2C1733817680422:(num 1733817681326) 2024-12-10T08:01:23,813 DEBUG [RS:0;80fbf959555f:39231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:23,813 INFO [RS:0;80fbf959555f:39231 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:23,813 INFO [RS:0;80fbf959555f:39231 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T08:01:23,814 INFO [RS:0;80fbf959555f:39231 {}] hbase.ChoreService(370): Chore service for: regionserver/80fbf959555f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T08:01:23,814 INFO [RS:0;80fbf959555f:39231 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T08:01:23,814 INFO [regionserver/80fbf959555f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T08:01:23,814 INFO [RS:0;80fbf959555f:39231 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39231 2024-12-10T08:01:23,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80fbf959555f,39231,1733817680422 2024-12-10T08:01:23,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:23,845 INFO [RS:0;80fbf959555f:39231 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T08:01:23,854 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80fbf959555f,39231,1733817680422] 2024-12-10T08:01:23,862 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80fbf959555f,39231,1733817680422 already deleted, retry=false 2024-12-10T08:01:23,862 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 80fbf959555f,39231,1733817680422 on 80fbf959555f,35517,1733817680282 2024-12-10T08:01:23,867 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 80fbf959555f,39231,1733817680422, splitWal=true, meta=true 2024-12-10T08:01:23,869 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 80fbf959555f,39231,1733817680422 (carryingMeta=true) 80fbf959555f,39231,1733817680422/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@72c1caa8[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-10T08:01:23,869 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 80fbf959555f,39231,1733817680422, splitWal=true, meta=true 2024-12-10T08:01:23,870 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 80fbf959555f,39231,1733817680422, splitWal=true, meta=true, isMeta: true 2024-12-10T08:01:23,871 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422-splitting 2024-12-10T08:01:23,871 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422-splitting dir is empty, no logs to split. 2024-12-10T08:01:23,871 INFO [PEWorker-1 {}] master.SplitWALManager(105): 80fbf959555f,39231,1733817680422 WAL count=0, meta=true 2024-12-10T08:01:23,874 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422-splitting dir is empty, no logs to split. 2024-12-10T08:01:23,874 INFO [PEWorker-1 {}] master.SplitWALManager(105): 80fbf959555f,39231,1733817680422 WAL count=0, meta=true 2024-12-10T08:01:23,874 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 80fbf959555f,39231,1733817680422 WAL splitting is done? wals=0, meta=true 2024-12-10T08:01:23,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T08:01:23,877 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T08:01:23,877 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T08:01:23,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:23,954 INFO [RS:0;80fbf959555f:39231 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T08:01:23,954 INFO [RS:0;80fbf959555f:39231 {}] regionserver.HRegionServer(1031): Exiting; stopping=80fbf959555f,39231,1733817680422; zookeeper connection closed. 2024-12-10T08:01:23,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39231-0x1000ecc00270001, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:23,955 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27987290 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27987290 2024-12-10T08:01:24,028 DEBUG [80fbf959555f:35517 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T08:01:24,028 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=80fbf959555f,46433,1733817680461 2024-12-10T08:01:24,029 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80fbf959555f,46433,1733817680461, state=OPENING 2024-12-10T08:01:24,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:24,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:24,045 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T08:01:24,045 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:24,045 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:24,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=80fbf959555f,46433,1733817680461}] 2024-12-10T08:01:24,096 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/80fbf959555f:0 server-side Connection retries=45 2024-12-10T08:01:24,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:24,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:24,096 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T08:01:24,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T08:01:24,096 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T08:01:24,096 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T08:01:24,096 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T08:01:24,097 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38129 2024-12-10T08:01:24,098 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38129 connecting to ZooKeeper ensemble=127.0.0.1:57631 2024-12-10T08:01:24,098 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:24,098 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:24,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381290x0, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T08:01:24,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-10T08:01:24,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38129-0x1000ecc00270003 connected 2024-12-10T08:01:24,129 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-10T08:01:24,130 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T08:01:24,131 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.cache 2024-12-10T08:01:24,131 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T08:01:24,131 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T08:01:24,131 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T08:01:24,131 INFO [Thread-209 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T08:01:24,131 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T08:01:24,132 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T08:01:24,132 INFO [Thread-209 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-10T08:01:24,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T08:01:24,133 INFO [Thread-209 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-10T08:01:24,134 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T08:01:24,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38129 2024-12-10T08:01:24,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38129 2024-12-10T08:01:24,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38129 2024-12-10T08:01:24,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38129 2024-12-10T08:01:24,139 DEBUG [Thread-209 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-10T08:01:24,139 INFO [Thread-209 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T08:01:24,139 INFO [Thread-209 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-10T08:01:24,139 INFO [Thread-209 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence is complete. 2024-12-10T08:01:24,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38129 2024-12-10T08:01:24,141 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(746): ClusterId : 04f5223b-7458-447f-92f5-e425d900141e 2024-12-10T08:01:24,142 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T08:01:24,153 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T08:01:24,153 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T08:01:24,163 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T08:01:24,163 DEBUG [RS:2;80fbf959555f:38129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a5eb5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=80fbf959555f/172.17.0.2:0 2024-12-10T08:01:24,173 DEBUG [RS:2;80fbf959555f:38129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;80fbf959555f:38129 2024-12-10T08:01:24,173 INFO [RS:2;80fbf959555f:38129 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T08:01:24,173 INFO [RS:2;80fbf959555f:38129 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T08:01:24,173 DEBUG [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T08:01:24,174 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(2659): reportForDuty to master=80fbf959555f,35517,1733817680282 with port=38129, startcode=1733817684095 2024-12-10T08:01:24,174 DEBUG [RS:2;80fbf959555f:38129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T08:01:24,176 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47077, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T08:01:24,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35517 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,176 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35517 {}] master.ServerManager(517): Registering regionserver=80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,178 DEBUG [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4 2024-12-10T08:01:24,178 DEBUG [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T08:01:24,178 DEBUG [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T08:01:24,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:24,187 DEBUG [RS:2;80fbf959555f:38129 {}] zookeeper.ZKUtil(111): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,187 WARN [RS:2;80fbf959555f:38129 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T08:01:24,187 INFO [RS:2;80fbf959555f:38129 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:24,187 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [80fbf959555f,38129,1733817684095] 2024-12-10T08:01:24,188 DEBUG [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,192 INFO [RS:2;80fbf959555f:38129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T08:01:24,195 INFO [RS:2;80fbf959555f:38129 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T08:01:24,196 INFO [RS:2;80fbf959555f:38129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T08:01:24,196 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,196 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T08:01:24,197 INFO [RS:2;80fbf959555f:38129 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T08:01:24,197 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,197 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,197 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,197 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,197 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,197 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,197 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/80fbf959555f:0, corePoolSize=2, maxPoolSize=2 2024-12-10T08:01:24,197 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,198 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,198 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,198 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,198 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,198 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/80fbf959555f:0, corePoolSize=1, maxPoolSize=1 2024-12-10T08:01:24,198 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T08:01:24,198 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:24,198 DEBUG [RS:2;80fbf959555f:38129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/80fbf959555f:0, corePoolSize=3, maxPoolSize=3 2024-12-10T08:01:24,199 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,199 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,199 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,199 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,199 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,199 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,38129,1733817684095-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T08:01:24,200 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36181, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T08:01:24,203 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T08:01:24,203 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T08:01:24,203 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T08:01:24,204 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C46433%2C1733817680461.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46433,1733817680461, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:24,216 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T08:01:24,216 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=80fbf959555f,38129,1733817684095-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,216 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,216 INFO [RS:2;80fbf959555f:38129 {}] regionserver.Replication(171): 80fbf959555f,38129,1733817684095 started 2024-12-10T08:01:24,220 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,46433,1733817680461/80fbf959555f%2C46433%2C1733817680461.meta.1733817684205.meta 2024-12-10T08:01:24,221 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:24,221 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:24,221 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T08:01:24,221 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T08:01:24,221 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T08:01:24,221 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T08:01:24,222 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:24,222 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T08:01:24,222 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T08:01:24,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T08:01:24,223 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T08:01:24,223 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:24,225 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/db79825b0e444d04adfd1c229266667b, delay=993 ms 2024-12-10T08:01:24,226 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/db79825b0e444d04adfd1c229266667b 2024-12-10T08:01:24,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:24,226 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T08:01:24,227 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T08:01:24,227 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:24,228 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/5fd43728d70e45e094bb7fc594fb8896, delay=936 ms 2024-12-10T08:01:24,229 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/5fd43728d70e45e094bb7fc594fb8896 2024-12-10T08:01:24,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:24,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T08:01:24,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T08:01:24,230 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:24,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:24,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T08:01:24,231 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T08:01:24,231 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:24,232 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T08:01:24,232 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(1482): Serving as 80fbf959555f,38129,1733817684095, RpcServer on 80fbf959555f/172.17.0.2:38129, sessionid=0x1000ecc00270003 2024-12-10T08:01:24,232 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T08:01:24,232 DEBUG [RS:2;80fbf959555f:38129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,232 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,38129,1733817684095' 2024-12-10T08:01:24,232 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T08:01:24,232 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/90333b1822ef473b910cd8a1db856df4, delay=990 ms 2024-12-10T08:01:24,233 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T08:01:24,233 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T08:01:24,234 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T08:01:24,234 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/90333b1822ef473b910cd8a1db856df4 2024-12-10T08:01:24,234 DEBUG [RS:2;80fbf959555f:38129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,234 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '80fbf959555f,38129,1733817684095' 2024-12-10T08:01:24,234 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T08:01:24,234 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T08:01:24,234 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T08:01:24,234 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T08:01:24,235 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:24,235 DEBUG [RS:2;80fbf959555f:38129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T08:01:24,235 INFO [RS:2;80fbf959555f:38129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T08:01:24,235 INFO [RS:2;80fbf959555f:38129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T08:01:24,235 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740 2024-12-10T08:01:24,235 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T08:01:24,235 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T08:01:24,236 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T08:01:24,236 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T08:01:24,237 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72419734, jitterRate=0.0791381299495697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T08:01:24,237 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T08:01:24,238 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733817684222Writing region info on filesystem at 1733817684222Initializing all the Stores at 1733817684222Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817684222Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817684222Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817684222Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733817684222Cleaning up temporary data from old regions at 1733817684235 (+13 ms)Running coprocessor post-open hooks at 1733817684237 (+2 ms)Region opened successfully at 1733817684237 2024-12-10T08:01:24,238 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733817684198 2024-12-10T08:01:24,240 DEBUG [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T08:01:24,240 INFO [RS_OPEN_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T08:01:24,241 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=80fbf959555f,46433,1733817680461 2024-12-10T08:01:24,242 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 80fbf959555f,46433,1733817680461, state=OPEN 2024-12-10T08:01:24,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:24,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:24,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T08:01:24,253 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=80fbf959555f,46433,1733817680461 2024-12-10T08:01:24,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:24,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:24,253 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T08:01:24,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-10T08:01:24,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=80fbf959555f,46433,1733817680461 in 208 msec 2024-12-10T08:01:24,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T08:01:24,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 380 msec 2024-12-10T08:01:24,258 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 80fbf959555f,39231,1733817680422 had 2 regions 2024-12-10T08:01:24,259 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 80fbf959555f,39231,1733817680422, splitWal=true, meta=true, isMeta: false 2024-12-10T08:01:24,260 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422-splitting dir is empty, no logs to split. 2024-12-10T08:01:24,260 INFO [PEWorker-3 {}] master.SplitWALManager(105): 80fbf959555f,39231,1733817680422 WAL count=0, meta=false 2024-12-10T08:01:24,261 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,39231,1733817680422-splitting dir is empty, no logs to split. 2024-12-10T08:01:24,261 INFO [PEWorker-3 {}] master.SplitWALManager(105): 80fbf959555f,39231,1733817680422 WAL count=0, meta=false 2024-12-10T08:01:24,261 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 80fbf959555f,39231,1733817680422 WAL splitting is done? wals=0, meta=false 2024-12-10T08:01:24,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN}] 2024-12-10T08:01:24,264 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN 2024-12-10T08:01:24,264 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T08:01:24,336 INFO [RS:2;80fbf959555f:38129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T08:01:24,336 INFO [RS:2;80fbf959555f:38129 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=80fbf959555f%2C38129%2C1733817684095, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,38129,1733817684095, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs, maxLogs=55 2024-12-10T08:01:24,358 INFO [RS:2;80fbf959555f:38129 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/WALs/80fbf959555f,38129,1733817684095/80fbf959555f%2C38129%2C1733817684095.1733817684337 2024-12-10T08:01:24,358 DEBUG [RS:2;80fbf959555f:38129 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T08:01:24,415 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(204): Hosts are {80fbf959555f=0} racks are {/default-rack=0} 2024-12-10T08:01:24,416 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T08:01:24,416 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T08:01:24,416 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T08:01:24,416 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T08:01:24,416 INFO [80fbf959555f:35517 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T08:01:24,416 INFO [80fbf959555f:35517 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T08:01:24,416 DEBUG [80fbf959555f:35517 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T08:01:24,417 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fd745abd220d6cf27defd98bdba14e28, regionState=OPENING, regionLocation=80fbf959555f,46433,1733817680461 2024-12-10T08:01:24,419 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 80fbf959555f:39231 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 80fbf959555f/172.17.0.2:39231 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T08:01:24,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=80fbf959555f,39231,1733817680422, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=80fbf959555f,39231,1733817680422, seqNum=-1, error=java.net.ConnectException: Call to address=80fbf959555f:39231 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 80fbf959555f/172.17.0.2:39231 2024-12-10T08:01:24,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=80fbf959555f,39231,1733817680422, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-10T08:01:24,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=80fbf959555f,39231,1733817680422, seqNum=-1 from cache 2024-12-10T08:01:24,425 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 80fbf959555f:39231 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 80fbf959555f/172.17.0.2:39231 2024-12-10T08:01:24,526 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T08:01:24,527 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=80fbf959555f,46433,1733817680461, seqNum=-1] 2024-12-10T08:01:24,527 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T08:01:24,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60271, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T08:01:24,531 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN because future has completed 2024-12-10T08:01:24,532 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd745abd220d6cf27defd98bdba14e28, server=80fbf959555f,46433,1733817680461}] 2024-12-10T08:01:24,582 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T08:01:24,582 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T08:01:24,584 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T08:01:24,584 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T08:01:24,585 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T08:01:24,585 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T08:01:24,688 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,688 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => fd745abd220d6cf27defd98bdba14e28, NAME => 'testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28.', STARTKEY => '', ENDKEY => ''} 2024-12-10T08:01:24,688 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733817681903 fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,688 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T08:01:24,689 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,689 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,689 INFO [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,690 INFO [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fd745abd220d6cf27defd98bdba14e28 columnFamilyName family 2024-12-10T08:01:24,690 DEBUG [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T08:01:24,691 DEBUG [StoreFileOpener-fd745abd220d6cf27defd98bdba14e28-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac, delay=939 ms 2024-12-10T08:01:24,692 DEBUG [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac 2024-12-10T08:01:24,692 INFO [StoreOpener-fd745abd220d6cf27defd98bdba14e28-1 {}] regionserver.HStore(327): Store=fd745abd220d6cf27defd98bdba14e28/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T08:01:24,693 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,694 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,694 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,695 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,695 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,696 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,696 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened fd745abd220d6cf27defd98bdba14e28; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61140945, jitterRate=-0.08892892301082611}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T08:01:24,696 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,697 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for fd745abd220d6cf27defd98bdba14e28: Running coprocessor pre-open hook at 1733817684689Writing region info on filesystem at 1733817684689Initializing all the Stores at 1733817684689Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733817684689Cleaning up temporary data from old regions at 1733817684695 (+6 ms)Running coprocessor post-open hooks at 1733817684696 (+1 ms)Region opened successfully at 1733817684697 (+1 ms) 2024-12-10T08:01:24,697 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28., pid=11, masterSystemTime=1733817684685 2024-12-10T08:01:24,700 DEBUG [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,700 INFO [RS_OPEN_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,700 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fd745abd220d6cf27defd98bdba14e28, regionState=OPEN, openSeqNum=10, regionLocation=80fbf959555f,46433,1733817680461 2024-12-10T08:01:24,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fd745abd220d6cf27defd98bdba14e28, server=80fbf959555f,46433,1733817680461 because future has completed 2024-12-10T08:01:24,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-10T08:01:24,705 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fd745abd220d6cf27defd98bdba14e28, server=80fbf959555f,46433,1733817680461 in 171 msec 2024-12-10T08:01:24,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-12-10T08:01:24,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733817681903, region=fd745abd220d6cf27defd98bdba14e28, ASSIGN in 443 msec 2024-12-10T08:01:24,707 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 80fbf959555f,39231,1733817680422 after splitting done 2024-12-10T08:01:24,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 80fbf959555f,39231,1733817680422, splitWal=true, meta=true in 844 msec 2024-12-10T08:01:24,732 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T08:01:24,732 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T08:01:24,732 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:24,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:24,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:24,733 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T08:01:24,733 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T08:01:24,733 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1408250445, stopped=false 2024-12-10T08:01:24,733 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=80fbf959555f,35517,1733817680282 2024-12-10T08:01:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T08:01:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:24,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:24,795 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T08:01:24,795 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T08:01:24,796 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:24,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:24,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:24,796 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80fbf959555f,46433,1733817680461' ***** 2024-12-10T08:01:24,796 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T08:01:24,797 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '80fbf959555f,38129,1733817684095' ***** 2024-12-10T08:01:24,797 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T08:01:24,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:24,797 INFO [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T08:01:24,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T08:01:24,797 INFO [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T08:01:24,797 INFO [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence 2024-12-10T08:01:24,797 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T08:01:24,797 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T08:01:24,797 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T08:01:24,797 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T08:01:24,798 INFO [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T08:01:24,798 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T08:01:24,798 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T08:01:24,798 INFO [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/ca914d58-2cd0-4e53-a20c-a7aecc61d7e3/bucket.persistence 2024-12-10T08:01:24,798 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T08:01:24,798 INFO [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T08:01:24,798 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T08:01:24,799 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T08:01:24,800 INFO [regionserver/80fbf959555f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:24,808 DEBUG [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1487): Thread RS:1;80fbf959555f:46433 started persisting bucket cache to file 2024-12-10T08:01:24,808 DEBUG [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:24,808 DEBUG [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:24,808 DEBUG [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1487): Thread RS:2;80fbf959555f:38129 started persisting bucket cache to file 2024-12-10T08:01:24,809 DEBUG [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T08:01:24,809 DEBUG [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T08:01:24,811 DEBUG [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:24,811 DEBUG [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817684808 2024-12-10T08:01:24,810 WARN [Thread-219 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-12-10T08:01:24,811 ERROR [RS:2;80fbf959555f:38129 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T08:01:24,811 DEBUG [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T08:01:24,811 DEBUG [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733817684808 2024-12-10T08:01:24,811 DEBUG [RS:2;80fbf959555f:38129 {}] bucket.BucketCache(1509): Thread RS:2;80fbf959555f:38129 finished persisting bucket cache to file, renaming 2024-12-10T08:01:24,811 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T08:01:24,811 INFO [RS:2;80fbf959555f:38129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T08:01:24,811 INFO [RS:2;80fbf959555f:38129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T08:01:24,812 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(959): stopping server 80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,812 INFO [RS:2;80fbf959555f:38129 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T08:01:24,812 INFO [RS:2;80fbf959555f:38129 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;80fbf959555f:38129. 2024-12-10T08:01:24,812 DEBUG [RS:2;80fbf959555f:38129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:24,812 DEBUG [RS:2;80fbf959555f:38129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:24,812 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(976): stopping server 80fbf959555f,38129,1733817684095; all regions closed. 2024-12-10T08:01:24,813 DEBUG [RS:2;80fbf959555f:38129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:24,813 INFO [RS:2;80fbf959555f:38129 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C38129%2C1733817684095:(num 1733817684337) 2024-12-10T08:01:24,813 DEBUG [RS:2;80fbf959555f:38129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:24,813 INFO [RS:2;80fbf959555f:38129 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:24,814 INFO [RS:2;80fbf959555f:38129 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T08:01:24,814 INFO [RS:2;80fbf959555f:38129 {}] hbase.ChoreService(370): Chore service for: regionserver/80fbf959555f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T08:01:24,814 INFO [RS:2;80fbf959555f:38129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T08:01:24,814 INFO [regionserver/80fbf959555f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T08:01:24,814 INFO [RS:2;80fbf959555f:38129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T08:01:24,814 INFO [RS:2;80fbf959555f:38129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T08:01:24,814 INFO [RS:2;80fbf959555f:38129 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T08:01:24,814 INFO [RS:2;80fbf959555f:38129 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38129 2024-12-10T08:01:24,825 DEBUG [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1509): Thread RS:1;80fbf959555f:46433 finished persisting bucket cache to file, renaming 2024-12-10T08:01:24,826 WARN [RS:1;80fbf959555f:46433 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T08:01:24,826 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T08:01:24,826 INFO [RS:1;80fbf959555f:46433 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T08:01:24,826 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T08:01:24,826 INFO [RS:1;80fbf959555f:46433 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T08:01:24,826 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(3091): Received CLOSE for fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,826 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(959): stopping server 80fbf959555f,46433,1733817680461 2024-12-10T08:01:24,826 INFO [RS:1;80fbf959555f:46433 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T08:01:24,826 INFO [RS:1;80fbf959555f:46433 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;80fbf959555f:46433. 2024-12-10T08:01:24,827 DEBUG [RS:1;80fbf959555f:46433 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T08:01:24,827 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T08:01:24,827 DEBUG [RS:1;80fbf959555f:46433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:24,827 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fd745abd220d6cf27defd98bdba14e28, disabling compactions & flushes 2024-12-10T08:01:24,827 INFO [RS:1;80fbf959555f:46433 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T08:01:24,827 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,827 INFO [RS:1;80fbf959555f:46433 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T08:01:24,827 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,827 INFO [RS:1;80fbf959555f:46433 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T08:01:24,827 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T08:01:24,827 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. after waiting 0 ms 2024-12-10T08:01:24,827 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,827 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T08:01:24,827 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1325): Online Regions={fd745abd220d6cf27defd98bdba14e28=testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28., 1588230740=hbase:meta,,1.1588230740} 2024-12-10T08:01:24,827 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T08:01:24,827 DEBUG [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, fd745abd220d6cf27defd98bdba14e28 2024-12-10T08:01:24,828 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T08:01:24,828 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T08:01:24,828 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T08:01:24,828 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T08:01:24,828 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T08:01:24,828 DEBUG [StoreFileCloser-fd745abd220d6cf27defd98bdba14e28-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/family/f3d8cb50d1c544d58e014ead3f9c3aac 2024-12-10T08:01:24,828 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-12-10T08:01:24,833 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/default/testBlockEvictionOnGracefulStop1733817681903/fd745abd220d6cf27defd98bdba14e28/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-10T08:01:24,834 INFO [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,834 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fd745abd220d6cf27defd98bdba14e28: Waiting for close lock at 1733817684827Running coprocessor pre-close hooks at 1733817684827Disabling compacts and flushes for region at 1733817684827Disabling writes for close at 1733817684827Writing region close event to WAL at 1733817684828 (+1 ms)Running coprocessor post-close hooks at 1733817684833 (+5 ms)Closed at 1733817684833 2024-12-10T08:01:24,834 DEBUG [RS_CLOSE_REGION-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28. 2024-12-10T08:01:24,839 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6621a5fb0fca429ebadddf507fc96559 2024-12-10T08:01:24,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80fbf959555f,38129,1733817684095 2024-12-10T08:01:24,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:24,849 INFO [RS:2;80fbf959555f:38129 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T08:01:24,850 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80fbf959555f,38129,1733817684095] 2024-12-10T08:01:24,853 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/6621a5fb0fca429ebadddf507fc96559 is 209, key is testBlockEvictionOnGracefulStop1733817681903,,1733817681907.fd745abd220d6cf27defd98bdba14e28./info:regioninfo/1733817684700/Put/seqid=0 2024-12-10T08:01:24,853 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/6621a5fb0fca429ebadddf507fc96559 2024-12-10T08:01:24,855 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/.tmp/info/6621a5fb0fca429ebadddf507fc96559 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/6621a5fb0fca429ebadddf507fc96559 2024-12-10T08:01:24,856 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/6621a5fb0fca429ebadddf507fc96559, delay=923 ms 2024-12-10T08:01:24,857 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/6621a5fb0fca429ebadddf507fc96559, entries=8, sequenceid=19, filesize=6.6 K 2024-12-10T08:01:24,858 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false 2024-12-10T08:01:24,859 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/db79825b0e444d04adfd1c229266667b 2024-12-10T08:01:24,859 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/info/6621a5fb0fca429ebadddf507fc96559 2024-12-10T08:01:24,860 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/ns/5fd43728d70e45e094bb7fc594fb8896 2024-12-10T08:01:24,860 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/table/90333b1822ef473b910cd8a1db856df4 2024-12-10T08:01:24,863 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-12-10T08:01:24,864 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T08:01:24,864 INFO [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:24,864 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733817684827Running coprocessor pre-close hooks at 1733817684827Disabling compacts and flushes for region at 1733817684828 (+1 ms)Disabling writes for close at 1733817684828Obtaining lock to block concurrent updates at 1733817684828Preparing flush snapshotting stores in 1588230740 at 1733817684828Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1733817684829 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733817684829Flushing 1588230740/info: creating writer at 1733817684829Flushing 1588230740/info: appending metadata at 1733817684852 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733817684853 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a52a574: reopening flushed file at 1733817684855 (+2 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 30ms, sequenceid=19, compaction requested=false at 1733817684858 (+3 ms)Writing region close event to WAL at 1733817684860 (+2 ms)Running coprocessor post-close hooks at 1733817684864 (+4 ms)Closed at 1733817684864 2024-12-10T08:01:24,864 DEBUG [RS_CLOSE_META-regionserver/80fbf959555f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T08:01:24,878 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80fbf959555f,38129,1733817684095 already deleted, retry=false 2024-12-10T08:01:24,878 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80fbf959555f,38129,1733817684095 expired; onlineServers=1 2024-12-10T08:01:24,893 INFO [regionserver/80fbf959555f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:24,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:24,969 INFO [RS:2;80fbf959555f:38129 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T08:01:24,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38129-0x1000ecc00270003, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:24,969 INFO [RS:2;80fbf959555f:38129 {}] regionserver.HRegionServer(1031): Exiting; stopping=80fbf959555f,38129,1733817684095; zookeeper connection closed. 2024-12-10T08:01:24,970 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@79dac47c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@79dac47c 2024-12-10T08:01:25,028 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(976): stopping server 80fbf959555f,46433,1733817680461; all regions closed. 2024-12-10T08:01:25,034 DEBUG [RS:1;80fbf959555f:46433 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:25,034 INFO [RS:1;80fbf959555f:46433 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C46433%2C1733817680461.meta:.meta(num 1733817684205) 2024-12-10T08:01:25,038 DEBUG [RS:1;80fbf959555f:46433 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/oldWALs 2024-12-10T08:01:25,038 INFO [RS:1;80fbf959555f:46433 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 80fbf959555f%2C46433%2C1733817680461:(num 1733817681331) 2024-12-10T08:01:25,038 DEBUG [RS:1;80fbf959555f:46433 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T08:01:25,038 INFO [RS:1;80fbf959555f:46433 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T08:01:25,038 INFO [RS:1;80fbf959555f:46433 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T08:01:25,039 INFO [RS:1;80fbf959555f:46433 {}] hbase.ChoreService(370): Chore service for: regionserver/80fbf959555f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T08:01:25,039 INFO [RS:1;80fbf959555f:46433 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T08:01:25,039 INFO [regionserver/80fbf959555f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T08:01:25,039 INFO [RS:1;80fbf959555f:46433 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46433 2024-12-10T08:01:25,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/80fbf959555f,46433,1733817680461 2024-12-10T08:01:25,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T08:01:25,062 INFO [RS:1;80fbf959555f:46433 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T08:01:25,070 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [80fbf959555f,46433,1733817680461] 2024-12-10T08:01:25,078 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/80fbf959555f,46433,1733817680461 already deleted, retry=false 2024-12-10T08:01:25,078 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 80fbf959555f,46433,1733817680461 expired; onlineServers=0 2024-12-10T08:01:25,078 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '80fbf959555f,35517,1733817680282' ***** 2024-12-10T08:01:25,078 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T08:01:25,079 INFO [M:0;80fbf959555f:35517 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T08:01:25,079 INFO [M:0;80fbf959555f:35517 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T08:01:25,079 DEBUG [M:0;80fbf959555f:35517 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T08:01:25,079 DEBUG [M:0;80fbf959555f:35517 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T08:01:25,079 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T08:01:25,079 DEBUG [master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.large.0-1733817681033 {}] cleaner.HFileCleaner(306): Exit Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.large.0-1733817681033,5,FailOnTimeoutGroup] 2024-12-10T08:01:25,079 DEBUG [master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.small.0-1733817681034 {}] cleaner.HFileCleaner(306): Exit Thread[master/80fbf959555f:0:becomeActiveMaster-HFileCleaner.small.0-1733817681034,5,FailOnTimeoutGroup] 2024-12-10T08:01:25,080 INFO [M:0;80fbf959555f:35517 {}] hbase.ChoreService(370): Chore service for: master/80fbf959555f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T08:01:25,080 INFO [M:0;80fbf959555f:35517 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T08:01:25,080 DEBUG [M:0;80fbf959555f:35517 {}] master.HMaster(1795): Stopping service threads 2024-12-10T08:01:25,080 INFO [M:0;80fbf959555f:35517 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T08:01:25,080 INFO [M:0;80fbf959555f:35517 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T08:01:25,081 INFO [M:0;80fbf959555f:35517 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T08:01:25,081 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T08:01:25,086 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T08:01:25,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T08:01:25,087 DEBUG [M:0;80fbf959555f:35517 {}] zookeeper.ZKUtil(347): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T08:01:25,087 WARN [M:0;80fbf959555f:35517 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T08:01:25,087 INFO [M:0;80fbf959555f:35517 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/.lastflushedseqids 2024-12-10T08:01:25,093 INFO [M:0;80fbf959555f:35517 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T08:01:25,094 INFO [M:0;80fbf959555f:35517 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T08:01:25,094 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T08:01:25,094 INFO [M:0;80fbf959555f:35517 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:25,094 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:25,094 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T08:01:25,094 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:25,094 INFO [M:0;80fbf959555f:35517 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-12-10T08:01:25,105 DEBUG [M:0;80fbf959555f:35517 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 977272b957184b95857701e7910c9562 2024-12-10T08:01:25,121 DEBUG [M:0;80fbf959555f:35517 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/977272b957184b95857701e7910c9562 is 82, key is hbase:meta,,1/info:regioninfo/1733817684241/Put/seqid=0 2024-12-10T08:01:25,121 INFO [M:0;80fbf959555f:35517 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/977272b957184b95857701e7910c9562 2024-12-10T08:01:25,128 DEBUG [M:0;80fbf959555f:35517 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 628ca0a194784207ab4f229959277671 2024-12-10T08:01:25,139 DEBUG [M:0;80fbf959555f:35517 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/628ca0a194784207ab4f229959277671 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733817682270/Put/seqid=0 2024-12-10T08:01:25,139 INFO [M:0;80fbf959555f:35517 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/628ca0a194784207ab4f229959277671 2024-12-10T08:01:25,145 DEBUG [M:0;80fbf959555f:35517 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0e415a2e8ad349aab29ee25fcedf3218 2024-12-10T08:01:25,157 DEBUG [M:0;80fbf959555f:35517 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0e415a2e8ad349aab29ee25fcedf3218 is 69, key is 80fbf959555f,38129,1733817684095/rs:state/1733817684177/Put/seqid=0 2024-12-10T08:01:25,157 INFO [M:0;80fbf959555f:35517 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0e415a2e8ad349aab29ee25fcedf3218 2024-12-10T08:01:25,159 INFO [M:0;80fbf959555f:35517 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e415a2e8ad349aab29ee25fcedf3218 2024-12-10T08:01:25,159 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/977272b957184b95857701e7910c9562 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/977272b957184b95857701e7910c9562 2024-12-10T08:01:25,160 INFO [M:0;80fbf959555f:35517 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/977272b957184b95857701e7910c9562, entries=14, sequenceid=102, filesize=5.9 K 2024-12-10T08:01:25,161 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/628ca0a194784207ab4f229959277671 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/628ca0a194784207ab4f229959277671 2024-12-10T08:01:25,162 INFO [M:0;80fbf959555f:35517 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/628ca0a194784207ab4f229959277671, entries=11, sequenceid=102, filesize=6.5 K 2024-12-10T08:01:25,162 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0e415a2e8ad349aab29ee25fcedf3218 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0e415a2e8ad349aab29ee25fcedf3218 2024-12-10T08:01:25,164 INFO [M:0;80fbf959555f:35517 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e415a2e8ad349aab29ee25fcedf3218 2024-12-10T08:01:25,164 INFO [M:0;80fbf959555f:35517 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/0decca1a-231c-93f5-ca8d-65a8aa3c744f/9e63494a-c89e-dba7-a204-346ba7b0d8d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0e415a2e8ad349aab29ee25fcedf3218, entries=3, sequenceid=102, filesize=5.3 K 2024-12-10T08:01:25,165 INFO [M:0;80fbf959555f:35517 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=102, compaction requested=false 2024-12-10T08:01:25,167 INFO [M:0;80fbf959555f:35517 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T08:01:25,167 DEBUG [M:0;80fbf959555f:35517 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733817685094Disabling compacts and flushes for region at 1733817685094Disabling writes for close at 1733817685094Obtaining lock to block concurrent updates at 1733817685094Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733817685094Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1733817685095 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733817685095Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733817685095Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733817685120 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733817685120Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733817685123 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733817685138 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733817685138Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733817685141 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733817685156 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733817685156Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@414c3348: reopening flushed file at 1733817685159 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59091e71: reopening flushed file at 1733817685160 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5143c7ca: reopening flushed file at 1733817685162 (+2 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=102, compaction requested=false at 1733817685165 (+3 ms)Writing region close event to WAL at 1733817685167 (+2 ms)Closed at 1733817685167 2024-12-10T08:01:25,168 INFO [M:0;80fbf959555f:35517 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T08:01:25,168 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T08:01:25,168 INFO [M:0;80fbf959555f:35517 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35517 2024-12-10T08:01:25,168 INFO [M:0;80fbf959555f:35517 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T08:01:25,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:25,170 INFO [RS:1;80fbf959555f:46433 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T08:01:25,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46433-0x1000ecc00270002, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:25,170 INFO [RS:1;80fbf959555f:46433 {}] regionserver.HRegionServer(1031): Exiting; stopping=80fbf959555f,46433,1733817680461; zookeeper connection closed. 2024-12-10T08:01:25,170 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75075248 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75075248 2024-12-10T08:01:25,171 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-10T08:01:25,295 INFO [M:0;80fbf959555f:35517 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T08:01:25,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:25,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35517-0x1000ecc00270000, quorum=127.0.0.1:57631, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T08:01:25,301 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T08:01:25,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T08:01:25,304 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T08:01:25,314 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=109 (was 119), ProcessCount=11 (was 11), AvailableMemoryMB=7027 (was 7109)