2024-11-21 14:48:16,378 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 14:48:16,398 main DEBUG Took 0.018010 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 14:48:16,398 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 14:48:16,398 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 14:48:16,399 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 14:48:16,401 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,409 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 14:48:16,423 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,425 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,426 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,426 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,427 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,427 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,429 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,429 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,430 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,430 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,432 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,432 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,433 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,434 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,435 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,437 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,438 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,439 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 14:48:16,440 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,440 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 14:48:16,443 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 14:48:16,444 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 14:48:16,447 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 14:48:16,448 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 14:48:16,449 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 14:48:16,450 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 14:48:16,462 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 14:48:16,465 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 14:48:16,467 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 14:48:16,467 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 14:48:16,467 main DEBUG createAppenders(={Console}) 2024-11-21 14:48:16,468 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-21 14:48:16,468 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-21 14:48:16,469 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-21 14:48:16,469 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 14:48:16,469 main DEBUG OutputStream closed 2024-11-21 14:48:16,470 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 14:48:16,470 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 14:48:16,470 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-21 14:48:16,542 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 14:48:16,544 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 14:48:16,545 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 14:48:16,546 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 14:48:16,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 14:48:16,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 14:48:16,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 14:48:16,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 14:48:16,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 14:48:16,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 14:48:16,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 14:48:16,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 14:48:16,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 14:48:16,549 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 14:48:16,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 14:48:16,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 14:48:16,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 14:48:16,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 14:48:16,553 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 14:48:16,554 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-21 14:48:16,554 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 14:48:16,555 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-21T14:48:16,570 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-21 14:48:16,573 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 14:48:16,573 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T14:48:16,917 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26 2024-11-21T14:48:16,948 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=195, ProcessCount=11, AvailableMemoryMB=3784 2024-11-21T14:48:17,077 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T14:48:17,189 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:17,204 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/cluster_3c0c08ea-b6bc-e2ec-8309-3ec0c8301f68, deleteOnExit=true 2024-11-21T14:48:17,306 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/cluster_3c0c08ea-b6bc-e2ec-8309-3ec0c8301f68/zookeeper_0, clientPort=58811, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/cluster_3c0c08ea-b6bc-e2ec-8309-3ec0c8301f68/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/cluster_3c0c08ea-b6bc-e2ec-8309-3ec0c8301f68/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T14:48:17,321 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58811 2024-11-21T14:48:17,322 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:17,422 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 with version=8 2024-11-21T14:48:17,423 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/hbase-staging 2024-11-21T14:48:17,533 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T14:48:17,837 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f7796871bd2a:0 server-side Connection retries=45 2024-11-21T14:48:17,849 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:17,850 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:17,858 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T14:48:17,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:17,859 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T14:48:18,002 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T14:48:18,095 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T14:48:18,134 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T14:48:18,146 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T14:48:18,150 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T14:48:18,193 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21123 (auto-detected) 2024-11-21T14:48:18,195 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T14:48:18,221 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45987 2024-11-21T14:48:18,256 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45987 connecting to ZooKeeper ensemble=127.0.0.1:58811 2024-11-21T14:48:18,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:459870x0, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T14:48:18,294 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45987-0x1001a8db9670000 connected 2024-11-21T14:48:18,322 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:18,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:18,333 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:18,337 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174, hbase.cluster.distributed=false 2024-11-21T14:48:18,361 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T14:48:18,368 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45987 2024-11-21T14:48:18,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45987 2024-11-21T14:48:18,369 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45987 2024-11-21T14:48:18,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45987 2024-11-21T14:48:18,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45987 2024-11-21T14:48:18,473 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7796871bd2a:0 server-side Connection retries=45 2024-11-21T14:48:18,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:18,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:18,476 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T14:48:18,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:18,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T14:48:18,480 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T14:48:18,484 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T14:48:18,485 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46863 2024-11-21T14:48:18,488 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46863 connecting to ZooKeeper ensemble=127.0.0.1:58811 2024-11-21T14:48:18,488 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:18,490 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:18,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468630x0, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T14:48:18,495 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46863-0x1001a8db9670001 connected 2024-11-21T14:48:18,495 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:18,501 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T14:48:18,514 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.cache 2024-11-21T14:48:18,520 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T14:48:18,521 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T14:48:18,523 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T14:48:18,525 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T14:48:18,525 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T14:48:18,532 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T14:48:18,539 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T14:48:18,539 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.persistence is complete. 2024-11-21T14:48:18,543 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T14:48:18,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T14:48:18,553 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T14:48:18,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46863 2024-11-21T14:48:18,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46863 2024-11-21T14:48:18,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46863 2024-11-21T14:48:18,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46863 2024-11-21T14:48:18,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46863 2024-11-21T14:48:18,594 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7796871bd2a:0 server-side Connection retries=45 2024-11-21T14:48:18,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:18,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:18,595 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T14:48:18,595 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:18,595 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T14:48:18,596 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T14:48:18,596 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T14:48:18,598 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43353 2024-11-21T14:48:18,600 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43353 connecting to ZooKeeper ensemble=127.0.0.1:58811 2024-11-21T14:48:18,601 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:18,603 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:18,607 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:433530x0, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T14:48:18,609 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:433530x0, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:18,609 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43353-0x1001a8db9670002 connected 2024-11-21T14:48:18,609 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T14:48:18,610 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.cache 2024-11-21T14:48:18,611 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T14:48:18,611 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T14:48:18,611 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T14:48:18,612 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T14:48:18,612 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T14:48:18,612 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T14:48:18,613 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T14:48:18,613 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.persistence is complete. 2024-11-21T14:48:18,614 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T14:48:18,616 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T14:48:18,618 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T14:48:18,619 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43353 2024-11-21T14:48:18,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43353 2024-11-21T14:48:18,623 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43353 2024-11-21T14:48:18,624 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43353 2024-11-21T14:48:18,625 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43353 2024-11-21T14:48:18,644 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f7796871bd2a:45987 2024-11-21T14:48:18,645 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f7796871bd2a,45987,1732200497622 2024-11-21T14:48:18,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:18,653 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:18,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:18,655 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f7796871bd2a,45987,1732200497622 2024-11-21T14:48:18,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T14:48:18,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:18,678 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T14:48:18,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:18,679 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:18,680 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T14:48:18,682 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f7796871bd2a,45987,1732200497622 from backup master directory 2024-11-21T14:48:18,685 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:18,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:18,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f7796871bd2a,45987,1732200497622 2024-11-21T14:48:18,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:18,686 WARN [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T14:48:18,687 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f7796871bd2a,45987,1732200497622 2024-11-21T14:48:18,689 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T14:48:18,691 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T14:48:18,701 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/hbase.id] with ID: 262dfd4c-2e8e-46a2-ab2d-9be0152fe865 2024-11-21T14:48:18,701 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.tmp/hbase.id 2024-11-21T14:48:18,707 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/hbase.id] 2024-11-21T14:48:18,748 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:18,753 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T14:48:18,773 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-21T14:48:18,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:18,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:18,777 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:18,807 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T14:48:18,811 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T14:48:18,862 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T14:48:18,869 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:18,937 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store 2024-11-21T14:48:18,962 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T14:48:18,967 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:18,969 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T14:48:18,970 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:18,970 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:18,972 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T14:48:18,973 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:18,973 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:18,974 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732200498969Disabling compacts and flushes for region at 1732200498969Disabling writes for close at 1732200498972 (+3 ms)Writing region close event to WAL at 1732200498973 (+1 ms)Closed at 1732200498973 2024-11-21T14:48:18,974 WARN [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/.initializing 2024-11-21T14:48:18,975 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/WALs/f7796871bd2a,45987,1732200497622 2024-11-21T14:48:18,990 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T14:48:19,004 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f7796871bd2a%2C45987%2C1732200497622, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/WALs/f7796871bd2a,45987,1732200497622, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/oldWALs, maxLogs=10 2024-11-21T14:48:19,048 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/WALs/f7796871bd2a,45987,1732200497622/f7796871bd2a%2C45987%2C1732200497622.1732200499009 2024-11-21T14:48:19,049 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:19,050 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:19,051 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:19,055 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,057 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,155 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T14:48:19,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:19,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T14:48:19,177 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,178 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:19,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,185 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T14:48:19,186 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:19,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T14:48:19,193 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,193 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:19,194 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,199 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,200 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,204 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,205 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,209 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T14:48:19,211 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:19,220 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T14:48:19,220 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66192261, jitterRate=-0.013658449053764343}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T14:48:19,230 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732200499075Initializing all the Stores at 1732200499076 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200499077 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200499078 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200499079 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200499079Cleaning up temporary data from old regions at 1732200499205 (+126 ms)Region opened successfully at 1732200499230 (+25 ms) 2024-11-21T14:48:19,233 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T14:48:19,275 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604b4acd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7796871bd2a/172.17.0.2:0 2024-11-21T14:48:19,313 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T14:48:19,329 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T14:48:19,329 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T14:48:19,334 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T14:48:19,335 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T14:48:19,343 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-21T14:48:19,343 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T14:48:19,378 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T14:48:19,389 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T14:48:19,392 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T14:48:19,394 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T14:48:19,396 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T14:48:19,398 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T14:48:19,400 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T14:48:19,405 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T14:48:19,407 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T14:48:19,408 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T14:48:19,410 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T14:48:19,433 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T14:48:19,435 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T14:48:19,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:19,452 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:19,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:19,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,454 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,457 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f7796871bd2a,45987,1732200497622, sessionid=0x1001a8db9670000, setting cluster-up flag (Was=false) 2024-11-21T14:48:19,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,470 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,477 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T14:48:19,480 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7796871bd2a,45987,1732200497622 2024-11-21T14:48:19,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,485 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:19,492 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T14:48:19,494 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7796871bd2a,45987,1732200497622 2024-11-21T14:48:19,500 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T14:48:19,532 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(746): ClusterId : 262dfd4c-2e8e-46a2-ab2d-9be0152fe865 2024-11-21T14:48:19,532 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(746): ClusterId : 262dfd4c-2e8e-46a2-ab2d-9be0152fe865 2024-11-21T14:48:19,534 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T14:48:19,534 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T14:48:19,541 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T14:48:19,541 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T14:48:19,541 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T14:48:19,541 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T14:48:19,548 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T14:48:19,549 DEBUG [RS:0;f7796871bd2a:46863 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d365d19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7796871bd2a/172.17.0.2:0 2024-11-21T14:48:19,551 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T14:48:19,551 DEBUG [RS:1;f7796871bd2a:43353 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bf0dbb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7796871bd2a/172.17.0.2:0 2024-11-21T14:48:19,572 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f7796871bd2a:43353 2024-11-21T14:48:19,575 DEBUG [RS:0;f7796871bd2a:46863 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f7796871bd2a:46863 2024-11-21T14:48:19,575 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T14:48:19,575 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T14:48:19,575 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T14:48:19,575 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T14:48:19,575 DEBUG [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T14:48:19,575 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T14:48:19,578 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7796871bd2a,45987,1732200497622 with port=46863, startcode=1732200498428 2024-11-21T14:48:19,579 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7796871bd2a,45987,1732200497622 with port=43353, startcode=1732200498593 2024-11-21T14:48:19,590 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:19,592 DEBUG [RS:0;f7796871bd2a:46863 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T14:48:19,592 DEBUG [RS:1;f7796871bd2a:43353 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T14:48:19,603 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T14:48:19,612 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T14:48:19,618 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f7796871bd2a,45987,1732200497622 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T14:48:19,628 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:19,628 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:19,628 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:19,629 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:19,629 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f7796871bd2a:0, corePoolSize=10, maxPoolSize=10 2024-11-21T14:48:19,630 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,630 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f7796871bd2a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T14:48:19,630 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,640 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:19,641 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T14:48:19,652 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,652 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T14:48:19,658 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732200529658 2024-11-21T14:48:19,660 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T14:48:19,661 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T14:48:19,665 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T14:48:19,666 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T14:48:19,666 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T14:48:19,666 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T14:48:19,667 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,670 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T14:48:19,672 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T14:48:19,673 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T14:48:19,676 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T14:48:19,676 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T14:48:19,678 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T14:48:19,678 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.large.0-1732200499678,5,FailOnTimeoutGroup] 2024-11-21T14:48:19,679 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:19,679 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.small.0-1732200499679,5,FailOnTimeoutGroup] 2024-11-21T14:48:19,679 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,680 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T14:48:19,681 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,681 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,691 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36901, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T14:48:19,691 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55567, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T14:48:19,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:19,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T14:48:19,698 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7796871bd2a,46863,1732200498428 2024-11-21T14:48:19,701 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45987 {}] master.ServerManager(517): Registering regionserver=f7796871bd2a,46863,1732200498428 2024-11-21T14:48:19,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T14:48:19,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:19,702 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T14:48:19,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T14:48:19,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,710 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:19,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T14:48:19,715 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45987 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7796871bd2a,43353,1732200498593 2024-11-21T14:48:19,715 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45987 {}] master.ServerManager(517): Registering regionserver=f7796871bd2a,43353,1732200498593 2024-11-21T14:48:19,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T14:48:19,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,717 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:19,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T14:48:19,720 DEBUG [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:19,720 DEBUG [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T14:48:19,720 DEBUG [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T14:48:19,720 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:19,720 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T14:48:19,721 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T14:48:19,725 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T14:48:19,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:19,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:19,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T14:48:19,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:19,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:19,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T14:48:19,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T14:48:19,727 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T14:48:19,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T14:48:19,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:19,731 DEBUG [RS:0;f7796871bd2a:46863 {}] zookeeper.ZKUtil(111): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7796871bd2a,46863,1732200498428 2024-11-21T14:48:19,731 WARN [RS:0;f7796871bd2a:46863 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T14:48:19,731 INFO [RS:0;f7796871bd2a:46863 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:19,731 DEBUG [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46863,1732200498428 2024-11-21T14:48:19,732 DEBUG [RS:1;f7796871bd2a:43353 {}] zookeeper.ZKUtil(111): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7796871bd2a,43353,1732200498593 2024-11-21T14:48:19,733 WARN [RS:1;f7796871bd2a:43353 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T14:48:19,733 INFO [RS:1;f7796871bd2a:43353 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:19,733 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,43353,1732200498593 2024-11-21T14:48:19,735 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7796871bd2a,43353,1732200498593] 2024-11-21T14:48:19,735 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7796871bd2a,46863,1732200498428] 2024-11-21T14:48:19,748 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T14:48:19,749 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59937617, jitterRate=-0.10685990750789642}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T14:48:19,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732200499692Initializing all the Stores at 1732200499693 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200499693Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200499694 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200499694Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200499694Cleaning up temporary data from old regions at 1732200499727 (+33 ms)Region opened successfully at 1732200499752 (+25 ms) 2024-11-21T14:48:19,752 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T14:48:19,753 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T14:48:19,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T14:48:19,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T14:48:19,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T14:48:19,754 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T14:48:19,755 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732200499752Disabling compacts and flushes for region at 1732200499752Disabling writes for close at 1732200499753 (+1 ms)Writing region close event to WAL at 1732200499754 (+1 ms)Closed at 1732200499754 2024-11-21T14:48:19,757 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:19,757 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T14:48:19,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T14:48:19,767 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T14:48:19,767 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T14:48:19,774 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T14:48:19,777 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T14:48:19,782 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T14:48:19,782 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T14:48:19,788 INFO [RS:0;f7796871bd2a:46863 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T14:48:19,788 INFO [RS:1;f7796871bd2a:43353 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T14:48:19,789 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,789 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,790 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T14:48:19,790 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T14:48:19,796 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T14:48:19,796 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T14:48:19,797 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,797 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,798 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,798 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7796871bd2a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T14:48:19,798 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7796871bd2a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T14:48:19,798 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,799 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:19,800 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:19,800 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:19,800 DEBUG [RS:1;f7796871bd2a:43353 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:19,800 DEBUG [RS:0;f7796871bd2a:46863 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:19,802 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,802 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,802 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,802 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,46863,1732200498428-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T14:48:19,803 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,43353,1732200498593-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T14:48:19,826 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T14:48:19,828 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,46863,1732200498428-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,828 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,829 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.Replication(171): f7796871bd2a,46863,1732200498428 started 2024-11-21T14:48:19,834 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T14:48:19,835 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,43353,1732200498593-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,835 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,835 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.Replication(171): f7796871bd2a,43353,1732200498593 started 2024-11-21T14:48:19,851 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,852 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(1482): Serving as f7796871bd2a,46863,1732200498428, RpcServer on f7796871bd2a/172.17.0.2:46863, sessionid=0x1001a8db9670001 2024-11-21T14:48:19,853 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T14:48:19,854 DEBUG [RS:0;f7796871bd2a:46863 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7796871bd2a,46863,1732200498428 2024-11-21T14:48:19,854 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,46863,1732200498428' 2024-11-21T14:48:19,854 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T14:48:19,856 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T14:48:19,857 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T14:48:19,857 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T14:48:19,857 DEBUG [RS:0;f7796871bd2a:46863 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7796871bd2a,46863,1732200498428 2024-11-21T14:48:19,857 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,46863,1732200498428' 2024-11-21T14:48:19,857 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T14:48:19,858 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T14:48:19,859 DEBUG [RS:0;f7796871bd2a:46863 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T14:48:19,859 INFO [RS:0;f7796871bd2a:46863 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T14:48:19,859 INFO [RS:0;f7796871bd2a:46863 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T14:48:19,864 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:19,864 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1482): Serving as f7796871bd2a,43353,1732200498593, RpcServer on f7796871bd2a/172.17.0.2:43353, sessionid=0x1001a8db9670002 2024-11-21T14:48:19,864 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T14:48:19,864 DEBUG [RS:1;f7796871bd2a:43353 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7796871bd2a,43353,1732200498593 2024-11-21T14:48:19,865 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,43353,1732200498593' 2024-11-21T14:48:19,865 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T14:48:19,866 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T14:48:19,867 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T14:48:19,867 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T14:48:19,867 DEBUG [RS:1;f7796871bd2a:43353 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7796871bd2a,43353,1732200498593 2024-11-21T14:48:19,867 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,43353,1732200498593' 2024-11-21T14:48:19,867 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T14:48:19,868 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T14:48:19,869 DEBUG [RS:1;f7796871bd2a:43353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T14:48:19,869 INFO [RS:1;f7796871bd2a:43353 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T14:48:19,869 INFO [RS:1;f7796871bd2a:43353 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T14:48:19,928 WARN [f7796871bd2a:45987 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T14:48:19,966 INFO [RS:0;f7796871bd2a:46863 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T14:48:19,967 INFO [RS:0;f7796871bd2a:46863 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7796871bd2a%2C46863%2C1732200498428, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46863,1732200498428, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs, maxLogs=55 2024-11-21T14:48:19,970 INFO [RS:1;f7796871bd2a:43353 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T14:48:19,971 INFO [RS:1;f7796871bd2a:43353 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7796871bd2a%2C43353%2C1732200498593, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,43353,1732200498593, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs, maxLogs=55 2024-11-21T14:48:19,992 INFO [RS:1;f7796871bd2a:43353 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,43353,1732200498593/f7796871bd2a%2C43353%2C1732200498593.1732200499973 2024-11-21T14:48:19,992 DEBUG [RS:1;f7796871bd2a:43353 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:19,994 INFO [RS:0;f7796871bd2a:46863 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46863,1732200498428/f7796871bd2a%2C46863%2C1732200498428.1732200499969 2024-11-21T14:48:19,995 DEBUG [RS:0;f7796871bd2a:46863 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:20,181 DEBUG [f7796871bd2a:45987 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T14:48:20,193 DEBUG [f7796871bd2a:45987 {}] balancer.BalancerClusterState(204): Hosts are {f7796871bd2a=0} racks are {/default-rack=0} 2024-11-21T14:48:20,196 DEBUG [f7796871bd2a:45987 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T14:48:20,197 DEBUG [f7796871bd2a:45987 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T14:48:20,197 DEBUG [f7796871bd2a:45987 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T14:48:20,197 DEBUG [f7796871bd2a:45987 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T14:48:20,197 INFO [f7796871bd2a:45987 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T14:48:20,197 INFO [f7796871bd2a:45987 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T14:48:20,197 DEBUG [f7796871bd2a:45987 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T14:48:20,204 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f7796871bd2a,43353,1732200498593 2024-11-21T14:48:20,210 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7796871bd2a,43353,1732200498593, state=OPENING 2024-11-21T14:48:20,215 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T14:48:20,217 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:20,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:20,217 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:20,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:20,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:20,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:20,220 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T14:48:20,222 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f7796871bd2a,43353,1732200498593}] 2024-11-21T14:48:20,397 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T14:48:20,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51353, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T14:48:20,411 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T14:48:20,411 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:20,412 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T14:48:20,413 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7796871bd2a%2C43353%2C1732200498593.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,43353,1732200498593, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs, maxLogs=55 2024-11-21T14:48:20,435 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,43353,1732200498593/f7796871bd2a%2C43353%2C1732200498593.meta.1732200500415.meta 2024-11-21T14:48:20,436 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:20,436 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:20,438 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T14:48:20,441 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T14:48:20,446 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T14:48:20,450 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T14:48:20,451 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:20,451 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T14:48:20,452 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T14:48:20,453 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T14:48:20,455 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T14:48:20,455 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:20,455 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:20,455 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T14:48:20,456 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T14:48:20,456 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:20,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:20,457 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T14:48:20,458 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T14:48:20,458 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:20,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:20,459 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T14:48:20,460 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T14:48:20,460 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:20,460 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:20,461 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T14:48:20,462 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:20,462 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:20,463 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T14:48:20,463 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T14:48:20,463 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T14:48:20,464 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T14:48:20,465 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58968452, jitterRate=-0.12130159139633179}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T14:48:20,465 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T14:48:20,467 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732200500452Writing region info on filesystem at 1732200500452Initializing all the Stores at 1732200500453 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200500453Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200500453Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200500453Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200500453Cleaning up temporary data from old regions at 1732200500463 (+10 ms)Running coprocessor post-open hooks at 1732200500465 (+2 ms)Region opened successfully at 1732200500467 (+2 ms) 2024-11-21T14:48:20,474 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732200500387 2024-11-21T14:48:20,486 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T14:48:20,486 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T14:48:20,488 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f7796871bd2a,43353,1732200498593 2024-11-21T14:48:20,489 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7796871bd2a,43353,1732200498593, state=OPEN 2024-11-21T14:48:20,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T14:48:20,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T14:48:20,492 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T14:48:20,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:20,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:20,493 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:20,493 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f7796871bd2a,43353,1732200498593 2024-11-21T14:48:20,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T14:48:20,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f7796871bd2a,43353,1732200498593 in 271 msec 2024-11-21T14:48:20,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T14:48:20,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 734 msec 2024-11-21T14:48:20,503 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:20,503 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T14:48:20,527 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T14:48:20,529 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7796871bd2a,43353,1732200498593, seqNum=-1] 2024-11-21T14:48:20,551 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T14:48:20,553 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44189, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T14:48:20,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0420 sec 2024-11-21T14:48:20,574 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732200500574, completionTime=-1 2024-11-21T14:48:20,577 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T14:48:20,577 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T14:48:20,604 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T14:48:20,604 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732200560604 2024-11-21T14:48:20,604 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732200620604 2024-11-21T14:48:20,604 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-21T14:48:20,606 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T14:48:20,613 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,45987,1732200497622-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:20,613 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,45987,1732200497622-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:20,613 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,45987,1732200497622-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:20,615 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f7796871bd2a:45987, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:20,615 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:20,615 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:20,623 DEBUG [master/f7796871bd2a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T14:48:20,645 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.958sec 2024-11-21T14:48:20,647 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T14:48:20,648 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T14:48:20,649 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T14:48:20,650 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T14:48:20,650 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T14:48:20,651 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,45987,1732200497622-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T14:48:20,651 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,45987,1732200497622-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T14:48:20,652 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T14:48:20,654 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T14:48:20,654 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,45987,1732200497622-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:20,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f5010e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T14:48:20,745 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T14:48:20,745 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T14:48:20,749 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f7796871bd2a,45987,-1 for getting cluster id 2024-11-21T14:48:20,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T14:48:20,761 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '262dfd4c-2e8e-46a2-ab2d-9be0152fe865' 2024-11-21T14:48:20,764 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T14:48:20,764 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "262dfd4c-2e8e-46a2-ab2d-9be0152fe865" 2024-11-21T14:48:20,767 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149feca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T14:48:20,767 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f7796871bd2a,45987,-1] 2024-11-21T14:48:20,769 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T14:48:20,771 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:20,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T14:48:20,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T14:48:20,776 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T14:48:20,784 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7796871bd2a,43353,1732200498593, seqNum=-1] 2024-11-21T14:48:20,784 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T14:48:20,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T14:48:20,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f7796871bd2a,45987,1732200497622 2024-11-21T14:48:20,811 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T14:48:20,816 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is f7796871bd2a,45987,1732200497622 2024-11-21T14:48:20,819 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@50aedccb 2024-11-21T14:48:20,819 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T14:48:20,822 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T14:48:20,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732200500807', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T14:48:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 2024-11-21T14:48:20,851 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T14:48:20,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732200500807" procId is: 4 2024-11-21T14:48:20,854 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:20,855 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T14:48:20,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:20,885 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 12f1f076bc9d26260885035618b640cd, NAME => 'testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732200500807', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:20,892 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:20,893 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(1722): Closing 12f1f076bc9d26260885035618b640cd, disabling compactions & flushes 2024-11-21T14:48:20,893 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:20,893 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:20,893 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. after waiting 0 ms 2024-11-21T14:48:20,893 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:20,893 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:20,893 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732200500807-pool-0 {}] regionserver.HRegion(1676): Region close journal for 12f1f076bc9d26260885035618b640cd: Waiting for close lock at 1732200500893Disabling compacts and flushes for region at 1732200500893Disabling writes for close at 1732200500893Writing region close event to WAL at 1732200500893Closed at 1732200500893 2024-11-21T14:48:20,895 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T14:48:20,902 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732200500896"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732200500896"}]},"ts":"1732200500896"} 2024-11-21T14:48:20,909 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T14:48:20,910 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T14:48:20,914 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732200500807","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732200500911"}]},"ts":"1732200500911"} 2024-11-21T14:48:20,919 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732200500807, state=ENABLING in hbase:meta 2024-11-21T14:48:20,920 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f7796871bd2a=0} racks are {/default-rack=0} 2024-11-21T14:48:20,920 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T14:48:20,920 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T14:48:20,920 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T14:48:20,921 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T14:48:20,921 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T14:48:20,921 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T14:48:20,921 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T14:48:20,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, ASSIGN}] 2024-11-21T14:48:20,925 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, ASSIGN 2024-11-21T14:48:20,926 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, ASSIGN; state=OFFLINE, location=f7796871bd2a,46863,1732200498428; forceNewPlan=false, retain=false 2024-11-21T14:48:20,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:21,079 INFO [f7796871bd2a:45987 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T14:48:21,080 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=12f1f076bc9d26260885035618b640cd, regionState=OPENING, regionLocation=f7796871bd2a,46863,1732200498428 2024-11-21T14:48:21,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, ASSIGN because future has completed 2024-11-21T14:48:21,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,46863,1732200498428}] 2024-11-21T14:48:21,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:21,238 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T14:48:21,240 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58553, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T14:48:21,243 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:21,243 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 12f1f076bc9d26260885035618b640cd, NAME => 'testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:21,244 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732200500807 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,244 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:21,244 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,244 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,245 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,249 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12f1f076bc9d26260885035618b640cd columnFamilyName family 2024-11-21T14:48:21,249 DEBUG [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:21,250 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.HStore(327): Store=12f1f076bc9d26260885035618b640cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:21,250 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,251 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,251 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,251 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,251 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,252 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,270 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T14:48:21,271 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 12f1f076bc9d26260885035618b640cd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74029276, jitterRate=0.10312217473983765}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T14:48:21,271 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:21,272 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 12f1f076bc9d26260885035618b640cd: Running coprocessor pre-open hook at 1732200501244Writing region info on filesystem at 1732200501244Initializing all the Stores at 1732200501245 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200501245Cleaning up temporary data from old regions at 1732200501251 (+6 ms)Running coprocessor post-open hooks at 1732200501271 (+20 ms)Region opened successfully at 1732200501272 (+1 ms) 2024-11-21T14:48:21,273 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd., pid=6, masterSystemTime=1732200501237 2024-11-21T14:48:21,277 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:21,277 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:21,278 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=12f1f076bc9d26260885035618b640cd, regionState=OPEN, openSeqNum=2, regionLocation=f7796871bd2a,46863,1732200498428 2024-11-21T14:48:21,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,46863,1732200498428 because future has completed 2024-11-21T14:48:21,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T14:48:21,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,46863,1732200498428 in 204 msec 2024-11-21T14:48:21,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T14:48:21,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, ASSIGN in 369 msec 2024-11-21T14:48:21,295 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T14:48:21,295 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732200500807","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732200501295"}]},"ts":"1732200501295"} 2024-11-21T14:48:21,299 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732200500807, state=ENABLED in hbase:meta 2024-11-21T14:48:21,300 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T14:48:21,302 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732200500807 in 456 msec 2024-11-21T14:48:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:21,511 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732200500807 completed 2024-11-21T14:48:21,512 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732200500807 get assigned. Timeout = 60000ms 2024-11-21T14:48:21,514 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T14:48:21,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732200500807 assigned to meta. Checking AM states. 2024-11-21T14:48:21,521 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T14:48:21,522 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732200500807 assigned. 2024-11-21T14:48:21,531 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732200500807', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd., hostname=f7796871bd2a,46863,1732200498428, seqNum=2] 2024-11-21T14:48:21,532 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T14:48:21,535 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T14:48:21,544 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 12f1f076bc9d26260885035618b640cd 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T14:48:21,570 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:21,608 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/.tmp/family/095e84f4107144fbbf1e335938c097fd is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T14:48:21,613 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/.tmp/family/095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:21,648 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/.tmp/family/095e84f4107144fbbf1e335938c097fd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:21,650 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd, delay=933 ms 2024-11-21T14:48:21,651 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T14:48:21,657 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 12f1f076bc9d26260885035618b640cd in 112ms, sequenceid=6, compaction requested=false 2024-11-21T14:48:21,658 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732200500807' 2024-11-21T14:48:21,660 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 12f1f076bc9d26260885035618b640cd: 2024-11-21T14:48:22,585 DEBUG [hfile-prefetch-1732200501650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 095e84f4107144fbbf1e335938c097fd_0. 2024-11-21T14:48:22,590 DEBUG [hfile-prefetch-1732200501650 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 095e84f4107144fbbf1e335938c097fd_117. 2024-11-21T14:48:22,592 DEBUG [hfile-prefetch-1732200501650 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd, with total blocks 2, and data blocks 1 2024-11-21T14:48:22,592 DEBUG [hfile-prefetch-1732200501650 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T14:48:22,592 DEBUG [hfile-prefetch-1732200501650 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T14:48:22,593 DEBUG [hfile-prefetch-1732200501650 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd. Will sleep 100ms and try the verification again. 2024-11-21T14:48:22,615 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T14:48:22,615 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T14:48:22,615 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:22,615 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T14:48:22,634 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T14:48:22,634 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732200502615 2024-11-21T14:48:22,634 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T14:48:22,671 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=12f1f076bc9d26260885035618b640cd, source=f7796871bd2a,46863,1732200498428, destination=f7796871bd2a,43353,1732200498593, warming up region on f7796871bd2a,43353,1732200498593 2024-11-21T14:48:22,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=12f1f076bc9d26260885035618b640cd, source=f7796871bd2a,46863,1732200498428, destination=f7796871bd2a,43353,1732200498593, running balancer 2024-11-21T14:48:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, REOPEN/MOVE 2024-11-21T14:48:22,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, REOPEN/MOVE 2024-11-21T14:48:22,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(7855): Warmup {ENCODED => 12f1f076bc9d26260885035618b640cd, NAME => 'testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:22,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:22,677 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:22,677 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12f1f076bc9d26260885035618b640cd, regionState=CLOSING, regionLocation=f7796871bd2a,46863,1732200498428 2024-11-21T14:48:22,678 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12f1f076bc9d26260885035618b640cd columnFamilyName family 2024-11-21T14:48:22,678 DEBUG [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:22,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, REOPEN/MOVE because future has completed 2024-11-21T14:48:22,681 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-21T14:48:22,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,46863,1732200498428}] 2024-11-21T14:48:22,682 DEBUG [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:22,683 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.HStore(327): Store=12f1f076bc9d26260885035618b640cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(1722): Closing 12f1f076bc9d26260885035618b640cd, disabling compactions & flushes 2024-11-21T14:48:22,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. after waiting 0 ms 2024-11-21T14:48:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,686 DEBUG [StoreFileCloser-12f1f076bc9d26260885035618b640cd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:22,688 WARN [hfile-prefetch-1732200501650 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-21T14:48:22,688 DEBUG [hfile-prefetch-1732200501650 {}] hfile.PrefetchExecutor(121): Prefetch completed for 095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:22,688 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43353 {}] regionserver.HRegion(1676): Region close journal for 12f1f076bc9d26260885035618b640cd: Waiting for close lock at 1732200502683Disabling compacts and flushes for region at 1732200502683Disabling writes for close at 1732200502683Writing region close event to WAL at 1732200502688 (+5 ms)Closed at 1732200502688 2024-11-21T14:48:22,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45987 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-21T14:48:22,840 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:22,840 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-21T14:48:22,841 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 12f1f076bc9d26260885035618b640cd, disabling compactions & flushes 2024-11-21T14:48:22,841 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,841 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,841 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. after waiting 0 ms 2024-11-21T14:48:22,841 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,843 DEBUG [StoreFileCloser-12f1f076bc9d26260885035618b640cd-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 095e84f4107144fbbf1e335938c097fd, starting offset: 0, end offset: 9223372036854775807 2024-11-21T14:48:22,844 DEBUG [StoreFileCloser-12f1f076bc9d26260885035618b640cd-family-1 {}] bucket.BucketCache(856): removed key 095e84f4107144fbbf1e335938c097fd_0 from back map with offset lock 2094080 in the evict process 2024-11-21T14:48:22,850 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T14:48:22,852 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:22,852 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 12f1f076bc9d26260885035618b640cd: Waiting for close lock at 1732200502841Running coprocessor pre-close hooks at 1732200502841Disabling compacts and flushes for region at 1732200502841Disabling writes for close at 1732200502841Writing region close event to WAL at 1732200502845 (+4 ms)Running coprocessor post-close hooks at 1732200502850 (+5 ms)Closed at 1732200502852 (+2 ms) 2024-11-21T14:48:22,852 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 12f1f076bc9d26260885035618b640cd move to f7796871bd2a,43353,1732200498593 record at close sequenceid=6 2024-11-21T14:48:22,855 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:22,856 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12f1f076bc9d26260885035618b640cd, regionState=CLOSED 2024-11-21T14:48:22,859 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,46863,1732200498428 because future has completed 2024-11-21T14:48:22,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T14:48:22,862 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,46863,1732200498428 in 179 msec 2024-11-21T14:48:22,862 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, REOPEN/MOVE; state=CLOSED, location=f7796871bd2a,43353,1732200498593; forceNewPlan=false, retain=false 2024-11-21T14:48:22,935 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T14:48:22,935 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T14:48:22,935 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:22,935 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T14:48:22,938 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T14:48:22,938 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T14:48:22,938 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:22,938 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T14:48:22,938 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T14:48:22,939 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732200502935 2024-11-21T14:48:22,939 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T14:48:22,941 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T14:48:22,941 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732200502938 2024-11-21T14:48:22,941 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T14:48:23,013 INFO [f7796871bd2a:45987 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T14:48:23,013 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12f1f076bc9d26260885035618b640cd, regionState=OPENING, regionLocation=f7796871bd2a,43353,1732200498593 2024-11-21T14:48:23,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, REOPEN/MOVE because future has completed 2024-11-21T14:48:23,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,43353,1732200498593}] 2024-11-21T14:48:23,172 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,173 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 12f1f076bc9d26260885035618b640cd, NAME => 'testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:23,173 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732200500807 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,173 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:23,174 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,174 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,175 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,176 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12f1f076bc9d26260885035618b640cd columnFamilyName family 2024-11-21T14:48:23,176 DEBUG [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:23,178 DEBUG [StoreFileOpener-12f1f076bc9d26260885035618b640cd-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd, delay=903 ms 2024-11-21T14:48:23,180 DEBUG [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:23,180 INFO [StoreOpener-12f1f076bc9d26260885035618b640cd-1 {}] regionserver.HStore(327): Store=12f1f076bc9d26260885035618b640cd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:23,180 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,181 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,181 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,181 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,181 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,186 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,187 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 12f1f076bc9d26260885035618b640cd; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71870930, jitterRate=0.07096031308174133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T14:48:23,187 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,188 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 12f1f076bc9d26260885035618b640cd: Running coprocessor pre-open hook at 1732200503174Writing region info on filesystem at 1732200503174Initializing all the Stores at 1732200503174Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200503174Cleaning up temporary data from old regions at 1732200503181 (+7 ms)Running coprocessor post-open hooks at 1732200503187 (+6 ms)Region opened successfully at 1732200503188 (+1 ms) 2024-11-21T14:48:23,189 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd., pid=9, masterSystemTime=1732200503170 2024-11-21T14:48:23,192 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,192 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,193 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=12f1f076bc9d26260885035618b640cd, regionState=OPEN, openSeqNum=10, regionLocation=f7796871bd2a,43353,1732200498593 2024-11-21T14:48:23,196 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,43353,1732200498593 because future has completed 2024-11-21T14:48:23,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-21T14:48:23,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 12f1f076bc9d26260885035618b640cd, server=f7796871bd2a,43353,1732200498593 in 180 msec 2024-11-21T14:48:23,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732200500807, region=12f1f076bc9d26260885035618b640cd, REOPEN/MOVE in 526 msec 2024-11-21T14:48:23,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T14:48:23,227 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T14:48:23,227 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:23,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:23,232 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:23,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T14:48:23,233 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T14:48:23,233 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-21T14:48:23,233 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f7796871bd2a,45987,1732200497622 2024-11-21T14:48:23,235 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:23,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:23,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:23,236 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,236 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T14:48:23,236 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T14:48:23,236 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:23,236 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:23,237 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:23,237 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7796871bd2a,46863,1732200498428' ***** 2024-11-21T14:48:23,237 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T14:48:23,237 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7796871bd2a,43353,1732200498593' ***** 2024-11-21T14:48:23,237 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T14:48:23,237 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:23,237 INFO [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T14:48:23,237 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:23,238 INFO [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T14:48:23,238 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T14:48:23,238 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T14:48:23,238 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T14:48:23,238 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T14:48:23,239 INFO [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.persistence 2024-11-21T14:48:23,239 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T14:48:23,239 INFO [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T14:48:23,239 INFO [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/bucket.persistence 2024-11-21T14:48:23,239 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T14:48:23,239 INFO [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T14:48:23,239 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T14:48:23,239 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T14:48:23,249 DEBUG [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1487): Thread RS:1;f7796871bd2a:43353 started persisting bucket cache to file 2024-11-21T14:48:23,249 DEBUG [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:23,249 DEBUG [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1487): Thread RS:0;f7796871bd2a:46863 started persisting bucket cache to file 2024-11-21T14:48:23,249 DEBUG [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T14:48:23,249 DEBUG [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:23,249 DEBUG [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T14:48:23,253 DEBUG [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T14:48:23,253 DEBUG [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T14:48:23,253 DEBUG [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732200503249 2024-11-21T14:48:23,253 DEBUG [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732200503249 2024-11-21T14:48:23,253 DEBUG [RS:0;f7796871bd2a:46863 {}] bucket.BucketCache(1509): Thread RS:0;f7796871bd2a:46863 finished persisting bucket cache to file, renaming 2024-11-21T14:48:23,253 DEBUG [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1509): Thread RS:1;f7796871bd2a:43353 finished persisting bucket cache to file, renaming 2024-11-21T14:48:23,253 WARN [RS:1;f7796871bd2a:43353 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-21T14:48:23,253 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T14:48:23,253 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T14:48:23,254 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T14:48:23,254 INFO [RS:0;f7796871bd2a:46863 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T14:48:23,254 INFO [RS:1;f7796871bd2a:43353 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T14:48:23,254 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T14:48:23,254 INFO [RS:1;f7796871bd2a:43353 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T14:48:23,254 INFO [RS:0;f7796871bd2a:46863 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T14:48:23,254 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(959): stopping server f7796871bd2a,46863,1732200498428 2024-11-21T14:48:23,254 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(3091): Received CLOSE for 12f1f076bc9d26260885035618b640cd 2024-11-21T14:48:23,254 INFO [RS:0;f7796871bd2a:46863 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T14:48:23,254 INFO [RS:0;f7796871bd2a:46863 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f7796871bd2a:46863. 2024-11-21T14:48:23,254 DEBUG [RS:0;f7796871bd2a:46863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:23,254 DEBUG [RS:0;f7796871bd2a:46863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:23,254 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(976): stopping server f7796871bd2a,46863,1732200498428; all regions closed. 2024-11-21T14:48:23,254 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(959): stopping server f7796871bd2a,43353,1732200498593 2024-11-21T14:48:23,254 INFO [RS:1;f7796871bd2a:43353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T14:48:23,255 INFO [RS:1;f7796871bd2a:43353 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f7796871bd2a:43353. 2024-11-21T14:48:23,255 DEBUG [RS:1;f7796871bd2a:43353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:23,255 DEBUG [RS:1;f7796871bd2a:43353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:23,255 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T14:48:23,255 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T14:48:23,255 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T14:48:23,255 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T14:48:23,255 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12f1f076bc9d26260885035618b640cd, disabling compactions & flushes 2024-11-21T14:48:23,256 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T14:48:23,256 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,256 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 12f1f076bc9d26260885035618b640cd=testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd.} 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,256 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. after waiting 0 ms 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,256 DEBUG [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1351): Waiting on 12f1f076bc9d26260885035618b640cd, 1588230740 2024-11-21T14:48:23,256 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T14:48:23,256 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T14:48:23,257 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-21T14:48:23,257 DEBUG [StoreFileCloser-12f1f076bc9d26260885035618b640cd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/family/095e84f4107144fbbf1e335938c097fd 2024-11-21T14:48:23,259 DEBUG [RS:0;f7796871bd2a:46863 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs 2024-11-21T14:48:23,260 INFO [RS:0;f7796871bd2a:46863 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7796871bd2a%2C46863%2C1732200498428:(num 1732200499969) 2024-11-21T14:48:23,260 DEBUG [RS:0;f7796871bd2a:46863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:23,260 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:23,260 INFO [RS:0;f7796871bd2a:46863 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T14:48:23,260 INFO [RS:0;f7796871bd2a:46863 {}] hbase.ChoreService(370): Chore service for: regionserver/f7796871bd2a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T14:48:23,261 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T14:48:23,261 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T14:48:23,261 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T14:48:23,261 INFO [RS:0;f7796871bd2a:46863 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T14:48:23,261 INFO [regionserver/f7796871bd2a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T14:48:23,261 INFO [RS:0;f7796871bd2a:46863 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46863 2024-11-21T14:48:23,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7796871bd2a,46863,1732200498428 2024-11-21T14:48:23,266 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:23,266 INFO [RS:0;f7796871bd2a:46863 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T14:48:23,267 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7796871bd2a,46863,1732200498428] 2024-11-21T14:48:23,267 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnRegionMove1732200500807/12f1f076bc9d26260885035618b640cd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T14:48:23,268 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,268 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12f1f076bc9d26260885035618b640cd: Waiting for close lock at 1732200503255Running coprocessor pre-close hooks at 1732200503255Disabling compacts and flushes for region at 1732200503255Disabling writes for close at 1732200503256 (+1 ms)Writing region close event to WAL at 1732200503258 (+2 ms)Running coprocessor post-close hooks at 1732200503268 (+10 ms)Closed at 1732200503268 2024-11-21T14:48:23,268 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd. 2024-11-21T14:48:23,269 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7796871bd2a,46863,1732200498428 already deleted, retry=false 2024-11-21T14:48:23,269 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7796871bd2a,46863,1732200498428 expired; onlineServers=1 2024-11-21T14:48:23,271 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d1a27f4f56eb425ca96ed5ed71b5d3c6 2024-11-21T14:48:23,288 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/info/d1a27f4f56eb425ca96ed5ed71b5d3c6 is 205, key is testBlockEvictionOnRegionMove1732200500807,,1732200500824.12f1f076bc9d26260885035618b640cd./info:regioninfo/1732200503193/Put/seqid=0 2024-11-21T14:48:23,289 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/info/d1a27f4f56eb425ca96ed5ed71b5d3c6 2024-11-21T14:48:23,300 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f01ccd412c724b3ca73a0bcd46a2df41 2024-11-21T14:48:23,308 INFO [regionserver/f7796871bd2a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:23,310 INFO [regionserver/f7796871bd2a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:23,314 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/ns/f01ccd412c724b3ca73a0bcd46a2df41 is 43, key is default/ns:d/1732200500557/Put/seqid=0 2024-11-21T14:48:23,314 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/ns/f01ccd412c724b3ca73a0bcd46a2df41 2024-11-21T14:48:23,324 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 722b40a6d09245a48ed609bd9e4f0303 2024-11-21T14:48:23,337 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/table/722b40a6d09245a48ed609bd9e4f0303 is 78, key is testBlockEvictionOnRegionMove1732200500807/table:state/1732200501295/Put/seqid=0 2024-11-21T14:48:23,338 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/table/722b40a6d09245a48ed609bd9e4f0303 2024-11-21T14:48:23,340 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/info/d1a27f4f56eb425ca96ed5ed71b5d3c6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/d1a27f4f56eb425ca96ed5ed71b5d3c6 2024-11-21T14:48:23,342 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/d1a27f4f56eb425ca96ed5ed71b5d3c6, delay=914 ms 2024-11-21T14:48:23,343 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/d1a27f4f56eb425ca96ed5ed71b5d3c6, entries=15, sequenceid=15, filesize=7.6 K 2024-11-21T14:48:23,343 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/ns/f01ccd412c724b3ca73a0bcd46a2df41 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f01ccd412c724b3ca73a0bcd46a2df41 2024-11-21T14:48:23,345 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f01ccd412c724b3ca73a0bcd46a2df41, delay=934 ms 2024-11-21T14:48:23,346 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f01ccd412c724b3ca73a0bcd46a2df41, entries=2, sequenceid=15, filesize=5.0 K 2024-11-21T14:48:23,346 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/table/722b40a6d09245a48ed609bd9e4f0303 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/722b40a6d09245a48ed609bd9e4f0303 2024-11-21T14:48:23,347 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/722b40a6d09245a48ed609bd9e4f0303, delay=935 ms 2024-11-21T14:48:23,348 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/722b40a6d09245a48ed609bd9e4f0303, entries=2, sequenceid=15, filesize=5.3 K 2024-11-21T14:48:23,349 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false 2024-11-21T14:48:23,349 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T14:48:23,351 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/d1a27f4f56eb425ca96ed5ed71b5d3c6 2024-11-21T14:48:23,352 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f01ccd412c724b3ca73a0bcd46a2df41 2024-11-21T14:48:23,356 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/722b40a6d09245a48ed609bd9e4f0303 2024-11-21T14:48:23,361 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-21T14:48:23,361 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T14:48:23,361 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T14:48:23,362 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732200503256Running coprocessor pre-close hooks at 1732200503256Disabling compacts and flushes for region at 1732200503256Disabling writes for close at 1732200503256Obtaining lock to block concurrent updates at 1732200503257 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732200503257Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732200503257Flushing stores of hbase:meta,,1.1588230740 at 1732200503258 (+1 ms)Flushing 1588230740/info: creating writer at 1732200503258Flushing 1588230740/info: appending metadata at 1732200503287 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732200503287Flushing 1588230740/ns: creating writer at 1732200503292 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732200503313 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1732200503313Flushing 1588230740/table: creating writer at 1732200503317 (+4 ms)Flushing 1588230740/table: appending metadata at 1732200503337 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732200503337Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d5bc014: reopening flushed file at 1732200503340 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6165d26: reopening flushed file at 1732200503343 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@753487c6: reopening flushed file at 1732200503346 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 93ms, sequenceid=15, compaction requested=false at 1732200503349 (+3 ms)Writing region close event to WAL at 1732200503356 (+7 ms)Running coprocessor post-close hooks at 1732200503361 (+5 ms)Closed at 1732200503361 2024-11-21T14:48:23,362 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T14:48:23,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:23,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46863-0x1001a8db9670001, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:23,370 INFO [RS:0;f7796871bd2a:46863 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T14:48:23,370 INFO [RS:0;f7796871bd2a:46863 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7796871bd2a,46863,1732200498428; zookeeper connection closed. 2024-11-21T14:48:23,370 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3afe0caa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3afe0caa 2024-11-21T14:48:23,456 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(976): stopping server f7796871bd2a,43353,1732200498593; all regions closed. 2024-11-21T14:48:23,459 DEBUG [RS:1;f7796871bd2a:43353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs 2024-11-21T14:48:23,459 INFO [RS:1;f7796871bd2a:43353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7796871bd2a%2C43353%2C1732200498593.meta:.meta(num 1732200500415) 2024-11-21T14:48:23,461 DEBUG [RS:1;f7796871bd2a:43353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs 2024-11-21T14:48:23,461 INFO [RS:1;f7796871bd2a:43353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7796871bd2a%2C43353%2C1732200498593:(num 1732200499973) 2024-11-21T14:48:23,461 DEBUG [RS:1;f7796871bd2a:43353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:23,461 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:23,461 INFO [RS:1;f7796871bd2a:43353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T14:48:23,461 INFO [RS:1;f7796871bd2a:43353 {}] hbase.ChoreService(370): Chore service for: regionserver/f7796871bd2a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T14:48:23,461 INFO [RS:1;f7796871bd2a:43353 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T14:48:23,461 INFO [regionserver/f7796871bd2a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T14:48:23,462 INFO [RS:1;f7796871bd2a:43353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43353 2024-11-21T14:48:23,464 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7796871bd2a,43353,1732200498593 2024-11-21T14:48:23,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:23,464 INFO [RS:1;f7796871bd2a:43353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T14:48:23,466 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7796871bd2a,43353,1732200498593] 2024-11-21T14:48:23,468 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7796871bd2a,43353,1732200498593 already deleted, retry=false 2024-11-21T14:48:23,468 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7796871bd2a,43353,1732200498593 expired; onlineServers=0 2024-11-21T14:48:23,468 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f7796871bd2a,45987,1732200497622' ***** 2024-11-21T14:48:23,468 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T14:48:23,468 INFO [M:0;f7796871bd2a:45987 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T14:48:23,468 INFO [M:0;f7796871bd2a:45987 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T14:48:23,469 DEBUG [M:0;f7796871bd2a:45987 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T14:48:23,469 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T14:48:23,469 DEBUG [M:0;f7796871bd2a:45987 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T14:48:23,469 DEBUG [master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.small.0-1732200499679 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.small.0-1732200499679,5,FailOnTimeoutGroup] 2024-11-21T14:48:23,469 DEBUG [master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.large.0-1732200499678 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.large.0-1732200499678,5,FailOnTimeoutGroup] 2024-11-21T14:48:23,469 INFO [M:0;f7796871bd2a:45987 {}] hbase.ChoreService(370): Chore service for: master/f7796871bd2a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T14:48:23,469 INFO [M:0;f7796871bd2a:45987 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T14:48:23,469 DEBUG [M:0;f7796871bd2a:45987 {}] master.HMaster(1795): Stopping service threads 2024-11-21T14:48:23,469 INFO [M:0;f7796871bd2a:45987 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T14:48:23,470 INFO [M:0;f7796871bd2a:45987 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T14:48:23,470 INFO [M:0;f7796871bd2a:45987 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T14:48:23,471 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T14:48:23,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T14:48:23,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,472 DEBUG [M:0;f7796871bd2a:45987 {}] zookeeper.ZKUtil(347): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T14:48:23,472 WARN [M:0;f7796871bd2a:45987 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T14:48:23,473 INFO [M:0;f7796871bd2a:45987 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.lastflushedseqids 2024-11-21T14:48:23,487 INFO [M:0;f7796871bd2a:45987 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T14:48:23,487 INFO [M:0;f7796871bd2a:45987 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T14:48:23,487 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T14:48:23,487 INFO [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:23,487 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:23,487 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T14:48:23,487 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:23,487 INFO [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-21T14:48:23,498 DEBUG [M:0;f7796871bd2a:45987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2b7c7d2765424cd6a1bb6b2ffced69b4 2024-11-21T14:48:23,512 DEBUG [M:0;f7796871bd2a:45987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b7c7d2765424cd6a1bb6b2ffced69b4 is 82, key is hbase:meta,,1/info:regioninfo/1732200500487/Put/seqid=0 2024-11-21T14:48:23,513 INFO [M:0;f7796871bd2a:45987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b7c7d2765424cd6a1bb6b2ffced69b4 2024-11-21T14:48:23,523 DEBUG [M:0;f7796871bd2a:45987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 44c8b6ad07d8442b81f015fd109d1a2e 2024-11-21T14:48:23,537 DEBUG [M:0;f7796871bd2a:45987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/44c8b6ad07d8442b81f015fd109d1a2e is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732200501302/Put/seqid=0 2024-11-21T14:48:23,538 INFO [M:0;f7796871bd2a:45987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/44c8b6ad07d8442b81f015fd109d1a2e 2024-11-21T14:48:23,547 DEBUG [M:0;f7796871bd2a:45987 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 722cda3dccdd4b6296136d1b2d4d402f 2024-11-21T14:48:23,560 DEBUG [M:0;f7796871bd2a:45987 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/722cda3dccdd4b6296136d1b2d4d402f is 69, key is f7796871bd2a,43353,1732200498593/rs:state/1732200499716/Put/seqid=0 2024-11-21T14:48:23,560 INFO [M:0;f7796871bd2a:45987 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/722cda3dccdd4b6296136d1b2d4d402f 2024-11-21T14:48:23,562 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2b7c7d2765424cd6a1bb6b2ffced69b4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b7c7d2765424cd6a1bb6b2ffced69b4 2024-11-21T14:48:23,565 INFO [M:0;f7796871bd2a:45987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2b7c7d2765424cd6a1bb6b2ffced69b4, entries=8, sequenceid=84, filesize=5.5 K 2024-11-21T14:48:23,565 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/44c8b6ad07d8442b81f015fd109d1a2e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/44c8b6ad07d8442b81f015fd109d1a2e 2024-11-21T14:48:23,566 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:23,566 INFO [RS:1;f7796871bd2a:43353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T14:48:23,566 DEBUG [pool-21-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43353-0x1001a8db9670002, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:23,566 INFO [RS:1;f7796871bd2a:43353 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7796871bd2a,43353,1732200498593; zookeeper connection closed. 2024-11-21T14:48:23,566 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@58876f30 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@58876f30 2024-11-21T14:48:23,567 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-21T14:48:23,567 INFO [M:0;f7796871bd2a:45987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/44c8b6ad07d8442b81f015fd109d1a2e, entries=9, sequenceid=84, filesize=6.5 K 2024-11-21T14:48:23,568 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/722cda3dccdd4b6296136d1b2d4d402f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/722cda3dccdd4b6296136d1b2d4d402f 2024-11-21T14:48:23,570 INFO [M:0;f7796871bd2a:45987 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/722cda3dccdd4b6296136d1b2d4d402f, entries=2, sequenceid=84, filesize=5.1 K 2024-11-21T14:48:23,571 INFO [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=84, compaction requested=false 2024-11-21T14:48:23,573 INFO [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:23,573 DEBUG [M:0;f7796871bd2a:45987 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732200503487Disabling compacts and flushes for region at 1732200503487Disabling writes for close at 1732200503487Obtaining lock to block concurrent updates at 1732200503487Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732200503487Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732200503488 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732200503488Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732200503488Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732200503512 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732200503512Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732200503515 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732200503537 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732200503537Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732200503540 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732200503559 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732200503559Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d2ee87e: reopening flushed file at 1732200503562 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a80c17b: reopening flushed file at 1732200503565 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c24cb7d: reopening flushed file at 1732200503568 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=84, compaction requested=false at 1732200503571 (+3 ms)Writing region close event to WAL at 1732200503573 (+2 ms)Closed at 1732200503573 2024-11-21T14:48:23,575 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T14:48:23,576 INFO [M:0;f7796871bd2a:45987 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T14:48:23,576 INFO [M:0;f7796871bd2a:45987 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45987 2024-11-21T14:48:23,576 INFO [M:0;f7796871bd2a:45987 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T14:48:23,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:23,679 INFO [M:0;f7796871bd2a:45987 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T14:48:23,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45987-0x1001a8db9670000, quorum=127.0.0.1:58811, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:23,683 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T14:48:23,705 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T14:48:23,705 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,714 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/f7796871bd2a:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f7796871bd2a:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732200501650 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732200503345 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/f7796871bd2a:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/f7796871bd2a:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1732200503342 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732200503178 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=196 (was 195) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3425 (was 3784) 2024-11-21T14:48:23,721 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=196, ProcessCount=11, AvailableMemoryMB=3425 2024-11-21T14:48:23,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/hadoop-log-dir so I do NOT create it in target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac 2024-11-21T14:48:23,721 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/hadoop-tmp-dir so I do NOT create it in target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac 2024-11-21T14:48:23,721 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,724 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/cluster_2f251695-3094-fc8d-ed90-d129a04777c9, deleteOnExit=true 2024-11-21T14:48:23,727 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/cluster_2f251695-3094-fc8d-ed90-d129a04777c9/zookeeper_0, clientPort=65150, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/cluster_2f251695-3094-fc8d-ed90-d129a04777c9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/cluster_2f251695-3094-fc8d-ed90-d129a04777c9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T14:48:23,729 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65150 2024-11-21T14:48:23,746 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 with version=8 2024-11-21T14:48:23,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/hbase-staging 2024-11-21T14:48:23,750 INFO [Time-limited test {}] client.ConnectionUtils(128): master/f7796871bd2a:0 server-side Connection retries=45 2024-11-21T14:48:23,750 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,750 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,750 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T14:48:23,750 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,750 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T14:48:23,751 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T14:48:23,751 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T14:48:23,751 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42929 2024-11-21T14:48:23,754 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42929 connecting to ZooKeeper ensemble=127.0.0.1:65150 2024-11-21T14:48:23,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429290x0, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T14:48:23,761 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42929-0x1001a8dd2950000 connected 2024-11-21T14:48:23,788 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,789 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,795 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:23,796 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174, hbase.cluster.distributed=false 2024-11-21T14:48:23,798 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T14:48:23,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42929 2024-11-21T14:48:23,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42929 2024-11-21T14:48:23,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42929 2024-11-21T14:48:23,804 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42929 2024-11-21T14:48:23,806 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42929 2024-11-21T14:48:23,824 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7796871bd2a:0 server-side Connection retries=45 2024-11-21T14:48:23,824 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,825 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T14:48:23,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,825 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T14:48:23,825 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T14:48:23,825 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T14:48:23,826 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40095 2024-11-21T14:48:23,828 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40095 connecting to ZooKeeper ensemble=127.0.0.1:65150 2024-11-21T14:48:23,828 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,828 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400950x0, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T14:48:23,832 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40095-0x1001a8dd2950001 connected 2024-11-21T14:48:23,833 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:23,833 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T14:48:23,834 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.cache 2024-11-21T14:48:23,834 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T14:48:23,835 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T14:48:23,835 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T14:48:23,836 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T14:48:23,836 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T14:48:23,837 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T14:48:23,837 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence is complete. 2024-11-21T14:48:23,840 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T14:48:23,844 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T14:48:23,845 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T14:48:23,847 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T14:48:23,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40095 2024-11-21T14:48:23,849 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40095 2024-11-21T14:48:23,853 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40095 2024-11-21T14:48:23,854 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40095 2024-11-21T14:48:23,856 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40095 2024-11-21T14:48:23,872 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7796871bd2a:0 server-side Connection retries=45 2024-11-21T14:48:23,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,872 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T14:48:23,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:23,872 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T14:48:23,872 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T14:48:23,873 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T14:48:23,873 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46135 2024-11-21T14:48:23,875 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46135 connecting to ZooKeeper ensemble=127.0.0.1:65150 2024-11-21T14:48:23,875 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,875 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461350x0, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T14:48:23,879 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461350x0, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:23,879 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46135-0x1001a8dd2950002 connected 2024-11-21T14:48:23,879 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T14:48:23,880 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.cache 2024-11-21T14:48:23,880 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T14:48:23,880 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T14:48:23,880 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T14:48:23,881 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T14:48:23,881 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-21T14:48:23,881 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T14:48:23,882 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T14:48:23,882 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence is complete. 2024-11-21T14:48:23,882 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T14:48:23,883 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T14:48:23,885 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T14:48:23,896 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46135 2024-11-21T14:48:23,899 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46135 2024-11-21T14:48:23,900 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46135 2024-11-21T14:48:23,906 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46135 2024-11-21T14:48:23,908 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46135 2024-11-21T14:48:23,924 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;f7796871bd2a:42929 2024-11-21T14:48:23,925 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/f7796871bd2a,42929,1732200503749 2024-11-21T14:48:23,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:23,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:23,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:23,928 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/f7796871bd2a,42929,1732200503749 2024-11-21T14:48:23,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T14:48:23,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T14:48:23,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,930 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,932 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T14:48:23,932 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/f7796871bd2a,42929,1732200503749 from backup master directory 2024-11-21T14:48:23,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:23,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:23,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/f7796871bd2a,42929,1732200503749 2024-11-21T14:48:23,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T14:48:23,934 WARN [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T14:48:23,934 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=f7796871bd2a,42929,1732200503749 2024-11-21T14:48:23,936 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/hbase.id] with ID: 46221a4c-d985-41d4-8427-7addb6ced2e6 2024-11-21T14:48:23,936 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.tmp/hbase.id 2024-11-21T14:48:23,944 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/hbase.id] 2024-11-21T14:48:23,987 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:23,987 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T14:48:23,989 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-21T14:48:23,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:23,997 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T14:48:23,998 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T14:48:23,998 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:24,021 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store 2024-11-21T14:48:24,028 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:24,028 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T14:48:24,028 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:24,028 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:24,028 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T14:48:24,028 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:24,028 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:24,028 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732200504028Disabling compacts and flushes for region at 1732200504028Disabling writes for close at 1732200504028Writing region close event to WAL at 1732200504028Closed at 1732200504028 2024-11-21T14:48:24,028 WARN [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/.initializing 2024-11-21T14:48:24,028 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/WALs/f7796871bd2a,42929,1732200503749 2024-11-21T14:48:24,034 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T14:48:24,038 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=f7796871bd2a%2C42929%2C1732200503749, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/WALs/f7796871bd2a,42929,1732200503749, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/oldWALs, maxLogs=10 2024-11-21T14:48:24,056 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/WALs/f7796871bd2a,42929,1732200503749/f7796871bd2a%2C42929%2C1732200503749.1732200504038 2024-11-21T14:48:24,057 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:24,057 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:24,057 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:24,057 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,057 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T14:48:24,069 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,070 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,074 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T14:48:24,074 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:24,075 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T14:48:24,078 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,078 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:24,079 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T14:48:24,083 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:24,083 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,084 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,084 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,085 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,085 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,085 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T14:48:24,086 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T14:48:24,092 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T14:48:24,092 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61677326, jitterRate=-0.08093622326850891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T14:48:24,094 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732200504058Initializing all the Stores at 1732200504058Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200504058Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200504059 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200504059Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200504059Cleaning up temporary data from old regions at 1732200504085 (+26 ms)Region opened successfully at 1732200504093 (+8 ms) 2024-11-21T14:48:24,094 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T14:48:24,099 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25800125, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7796871bd2a/172.17.0.2:0 2024-11-21T14:48:24,100 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T14:48:24,100 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T14:48:24,100 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T14:48:24,100 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T14:48:24,100 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T14:48:24,101 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-21T14:48:24,101 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T14:48:24,102 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T14:48:24,103 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T14:48:24,105 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T14:48:24,105 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T14:48:24,106 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T14:48:24,107 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T14:48:24,107 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T14:48:24,108 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T14:48:24,109 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T14:48:24,110 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T14:48:24,112 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T14:48:24,114 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T14:48:24,115 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T14:48:24,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:24,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:24,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:24,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,117 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,118 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=f7796871bd2a,42929,1732200503749, sessionid=0x1001a8dd2950000, setting cluster-up flag (Was=false) 2024-11-21T14:48:24,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,126 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T14:48:24,127 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7796871bd2a,42929,1732200503749 2024-11-21T14:48:24,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,135 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T14:48:24,137 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=f7796871bd2a,42929,1732200503749 2024-11-21T14:48:24,137 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T14:48:24,139 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:24,139 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T14:48:24,139 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T14:48:24,139 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: f7796871bd2a,42929,1732200503749 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/f7796871bd2a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/f7796871bd2a:0, corePoolSize=10, maxPoolSize=10 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/f7796871bd2a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T14:48:24,141 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,145 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:24,145 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T14:48:24,145 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,146 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732200534147 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T14:48:24,147 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,148 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T14:48:24,148 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T14:48:24,148 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T14:48:24,152 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T14:48:24,152 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T14:48:24,152 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.large.0-1732200504152,5,FailOnTimeoutGroup] 2024-11-21T14:48:24,152 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.small.0-1732200504152,5,FailOnTimeoutGroup] 2024-11-21T14:48:24,152 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,152 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T14:48:24,152 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,153 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,164 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T14:48:24,164 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:24,171 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:24,172 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T14:48:24,175 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T14:48:24,175 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,176 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T14:48:24,179 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T14:48:24,179 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,180 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T14:48:24,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T14:48:24,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,183 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T14:48:24,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T14:48:24,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,188 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,188 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T14:48:24,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:24,189 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:24,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T14:48:24,190 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T14:48:24,190 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T14:48:24,191 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T14:48:24,197 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T14:48:24,198 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60648887, jitterRate=-0.0962611585855484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T14:48:24,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732200504171Initializing all the Stores at 1732200504171Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200504171Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200504172 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200504172Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200504172Cleaning up temporary data from old regions at 1732200504190 (+18 ms)Region opened successfully at 1732200504199 (+9 ms) 2024-11-21T14:48:24,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T14:48:24,199 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T14:48:24,199 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T14:48:24,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T14:48:24,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T14:48:24,200 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T14:48:24,200 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732200504199Disabling compacts and flushes for region at 1732200504199Disabling writes for close at 1732200504200 (+1 ms)Writing region close event to WAL at 1732200504200Closed at 1732200504200 2024-11-21T14:48:24,202 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:24,202 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T14:48:24,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T14:48:24,203 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T14:48:24,204 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T14:48:24,211 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(746): ClusterId : 46221a4c-d985-41d4-8427-7addb6ced2e6 2024-11-21T14:48:24,212 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(746): ClusterId : 46221a4c-d985-41d4-8427-7addb6ced2e6 2024-11-21T14:48:24,212 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T14:48:24,212 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T14:48:24,214 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T14:48:24,214 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T14:48:24,214 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T14:48:24,214 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T14:48:24,217 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T14:48:24,217 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T14:48:24,217 DEBUG [RS:1;f7796871bd2a:46135 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff3d236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7796871bd2a/172.17.0.2:0 2024-11-21T14:48:24,218 DEBUG [RS:0;f7796871bd2a:40095 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9f4cb3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7796871bd2a/172.17.0.2:0 2024-11-21T14:48:24,230 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;f7796871bd2a:46135 2024-11-21T14:48:24,230 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T14:48:24,230 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T14:48:24,230 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T14:48:24,231 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7796871bd2a,42929,1732200503749 with port=46135, startcode=1732200503872 2024-11-21T14:48:24,232 DEBUG [RS:1;f7796871bd2a:46135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T14:48:24,235 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59013, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T14:48:24,235 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,236 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(517): Registering regionserver=f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,236 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;f7796871bd2a:40095 2024-11-21T14:48:24,237 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T14:48:24,237 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T14:48:24,237 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T14:48:24,237 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:24,238 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7796871bd2a,42929,1732200503749 with port=40095, startcode=1732200503824 2024-11-21T14:48:24,238 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T14:48:24,238 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T14:48:24,238 DEBUG [RS:0;f7796871bd2a:40095 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T14:48:24,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:24,240 DEBUG [RS:1;f7796871bd2a:46135 {}] zookeeper.ZKUtil(111): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,240 WARN [RS:1;f7796871bd2a:46135 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T14:48:24,240 INFO [RS:1;f7796871bd2a:46135 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:24,241 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,241 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54823, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T14:48:24,241 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7796871bd2a,46135,1732200503872] 2024-11-21T14:48:24,241 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7796871bd2a,40095,1732200503824 2024-11-21T14:48:24,241 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(517): Registering regionserver=f7796871bd2a,40095,1732200503824 2024-11-21T14:48:24,243 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:24,243 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T14:48:24,243 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T14:48:24,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:24,248 DEBUG [RS:0;f7796871bd2a:40095 {}] zookeeper.ZKUtil(111): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7796871bd2a,40095,1732200503824 2024-11-21T14:48:24,248 WARN [RS:0;f7796871bd2a:40095 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T14:48:24,248 INFO [RS:0;f7796871bd2a:40095 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:24,248 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7796871bd2a,40095,1732200503824] 2024-11-21T14:48:24,248 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,40095,1732200503824 2024-11-21T14:48:24,249 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T14:48:24,252 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T14:48:24,253 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T14:48:24,256 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T14:48:24,256 INFO [RS:1;f7796871bd2a:46135 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T14:48:24,256 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,257 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T14:48:24,257 INFO [RS:0;f7796871bd2a:40095 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T14:48:24,257 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,257 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T14:48:24,257 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,257 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T14:48:24,258 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T14:48:24,258 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,258 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7796871bd2a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T14:48:24,258 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,258 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7796871bd2a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:24,259 DEBUG [RS:0;f7796871bd2a:40095 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:24,259 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:24,260 DEBUG [RS:1;f7796871bd2a:46135 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:24,260 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,260 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,260 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,260 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,260 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,260 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,260 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,40095,1732200503824-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T14:48:24,261 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,261 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,261 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,261 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,261 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,46135,1732200503872-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T14:48:24,279 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T14:48:24,279 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,46135,1732200503872-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,279 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,279 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.Replication(171): f7796871bd2a,46135,1732200503872 started 2024-11-21T14:48:24,281 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T14:48:24,281 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,40095,1732200503824-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,281 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,281 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.Replication(171): f7796871bd2a,40095,1732200503824 started 2024-11-21T14:48:24,294 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,294 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1482): Serving as f7796871bd2a,46135,1732200503872, RpcServer on f7796871bd2a/172.17.0.2:46135, sessionid=0x1001a8dd2950002 2024-11-21T14:48:24,294 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T14:48:24,294 DEBUG [RS:1;f7796871bd2a:46135 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,294 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,46135,1732200503872' 2024-11-21T14:48:24,294 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T14:48:24,295 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T14:48:24,295 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T14:48:24,295 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T14:48:24,295 DEBUG [RS:1;f7796871bd2a:46135 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,295 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,46135,1732200503872' 2024-11-21T14:48:24,295 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T14:48:24,296 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T14:48:24,296 DEBUG [RS:1;f7796871bd2a:46135 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T14:48:24,296 INFO [RS:1;f7796871bd2a:46135 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T14:48:24,296 INFO [RS:1;f7796871bd2a:46135 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T14:48:24,301 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,301 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1482): Serving as f7796871bd2a,40095,1732200503824, RpcServer on f7796871bd2a/172.17.0.2:40095, sessionid=0x1001a8dd2950001 2024-11-21T14:48:24,301 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T14:48:24,301 DEBUG [RS:0;f7796871bd2a:40095 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7796871bd2a,40095,1732200503824 2024-11-21T14:48:24,302 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,40095,1732200503824' 2024-11-21T14:48:24,302 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T14:48:24,302 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T14:48:24,303 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T14:48:24,303 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T14:48:24,303 DEBUG [RS:0;f7796871bd2a:40095 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7796871bd2a,40095,1732200503824 2024-11-21T14:48:24,303 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,40095,1732200503824' 2024-11-21T14:48:24,303 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T14:48:24,303 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T14:48:24,304 DEBUG [RS:0;f7796871bd2a:40095 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T14:48:24,304 INFO [RS:0;f7796871bd2a:40095 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T14:48:24,304 INFO [RS:0;f7796871bd2a:40095 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T14:48:24,354 WARN [f7796871bd2a:42929 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T14:48:24,397 INFO [RS:1;f7796871bd2a:46135 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T14:48:24,398 INFO [RS:1;f7796871bd2a:46135 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7796871bd2a%2C46135%2C1732200503872, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46135,1732200503872, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs, maxLogs=55 2024-11-21T14:48:24,404 INFO [RS:0;f7796871bd2a:40095 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T14:48:24,405 INFO [RS:0;f7796871bd2a:40095 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7796871bd2a%2C40095%2C1732200503824, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,40095,1732200503824, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs, maxLogs=55 2024-11-21T14:48:24,420 INFO [RS:1;f7796871bd2a:46135 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46135,1732200503872/f7796871bd2a%2C46135%2C1732200503872.1732200504401 2024-11-21T14:48:24,421 DEBUG [RS:1;f7796871bd2a:46135 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:24,424 INFO [RS:0;f7796871bd2a:40095 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,40095,1732200503824/f7796871bd2a%2C40095%2C1732200503824.1732200504406 2024-11-21T14:48:24,424 DEBUG [RS:0;f7796871bd2a:40095 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:24,605 DEBUG [f7796871bd2a:42929 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T14:48:24,605 DEBUG [f7796871bd2a:42929 {}] balancer.BalancerClusterState(204): Hosts are {f7796871bd2a=0} racks are {/default-rack=0} 2024-11-21T14:48:24,606 DEBUG [f7796871bd2a:42929 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T14:48:24,606 DEBUG [f7796871bd2a:42929 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T14:48:24,606 DEBUG [f7796871bd2a:42929 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T14:48:24,606 DEBUG [f7796871bd2a:42929 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T14:48:24,606 INFO [f7796871bd2a:42929 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T14:48:24,606 INFO [f7796871bd2a:42929 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T14:48:24,606 DEBUG [f7796871bd2a:42929 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T14:48:24,607 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,608 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7796871bd2a,46135,1732200503872, state=OPENING 2024-11-21T14:48:24,610 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T14:48:24,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:24,612 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T14:48:24,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=f7796871bd2a,46135,1732200503872}] 2024-11-21T14:48:24,612 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:24,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:24,613 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:24,765 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T14:48:24,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53263, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T14:48:24,771 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T14:48:24,771 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:24,771 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T14:48:24,772 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7796871bd2a%2C46135%2C1732200503872.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46135,1732200503872, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs, maxLogs=55 2024-11-21T14:48:24,791 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,46135,1732200503872/f7796871bd2a%2C46135%2C1732200503872.meta.1732200504773.meta 2024-11-21T14:48:24,792 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:24,792 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:24,792 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T14:48:24,792 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T14:48:24,792 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T14:48:24,793 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T14:48:24,793 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:24,793 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T14:48:24,793 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T14:48:24,794 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-21T14:48:24,795 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T14:48:24,795 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,795 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-21T14:48:24,796 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T14:48:24,796 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,796 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-21T14:48:24,797 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T14:48:24,797 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,797 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-21T14:48:24,798 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T14:48:24,798 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T14:48:24,798 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T14:48:24,799 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:24,799 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740 2024-11-21T14:48:24,799 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T14:48:24,800 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T14:48:24,800 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T14:48:24,801 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T14:48:24,801 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60562276, jitterRate=-0.09755176305770874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T14:48:24,801 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T14:48:24,802 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732200504793Writing region info on filesystem at 1732200504793Initializing all the Stores at 1732200504793Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200504793Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200504794 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200504794Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732200504794Cleaning up temporary data from old regions at 1732200504800 (+6 ms)Running coprocessor post-open hooks at 1732200504801 (+1 ms)Region opened successfully at 1732200504802 (+1 ms) 2024-11-21T14:48:24,803 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732200504765 2024-11-21T14:48:24,806 DEBUG [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T14:48:24,806 INFO [RS_OPEN_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T14:48:24,806 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,807 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as f7796871bd2a,46135,1732200503872, state=OPEN 2024-11-21T14:48:24,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T14:48:24,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T14:48:24,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:24,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T14:48:24,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:24,809 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=f7796871bd2a,46135,1732200503872 2024-11-21T14:48:24,809 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T14:48:24,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T14:48:24,811 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=f7796871bd2a,46135,1732200503872 in 197 msec 2024-11-21T14:48:24,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T14:48:24,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec 2024-11-21T14:48:24,813 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T14:48:24,813 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T14:48:24,817 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T14:48:24,817 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7796871bd2a,46135,1732200503872, seqNum=-1] 2024-11-21T14:48:24,817 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T14:48:24,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59355, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T14:48:24,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 685 msec 2024-11-21T14:48:24,825 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732200504825, completionTime=-1 2024-11-21T14:48:24,825 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-21T14:48:24,825 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T14:48:24,827 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T14:48:24,827 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732200564827 2024-11-21T14:48:24,827 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732200624827 2024-11-21T14:48:24,827 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-21T14:48:24,828 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,42929,1732200503749-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,828 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,42929,1732200503749-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,828 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,42929,1732200503749-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,829 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-f7796871bd2a:42929, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,829 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,829 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,831 DEBUG [master/f7796871bd2a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.899sec 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,42929,1732200503749-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T14:48:24,834 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,42929,1732200503749-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T14:48:24,835 DEBUG [master/f7796871bd2a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T14:48:24,835 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T14:48:24,835 INFO [master/f7796871bd2a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,42929,1732200503749-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:24,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T14:48:24,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request f7796871bd2a,42929,-1 for getting cluster id 2024-11-21T14:48:24,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T14:48:24,914 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '46221a4c-d985-41d4-8427-7addb6ced2e6' 2024-11-21T14:48:24,915 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T14:48:24,915 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "46221a4c-d985-41d4-8427-7addb6ced2e6" 2024-11-21T14:48:24,915 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cf5e24a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T14:48:24,915 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [f7796871bd2a,42929,-1] 2024-11-21T14:48:24,916 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T14:48:24,916 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:24,918 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57820, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T14:48:24,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T14:48:24,920 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T14:48:24,921 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=f7796871bd2a,46135,1732200503872, seqNum=-1] 2024-11-21T14:48:24,921 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T14:48:24,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44676, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T14:48:24,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=f7796871bd2a,42929,1732200503749 2024-11-21T14:48:24,926 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T14:48:24,927 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is f7796871bd2a,42929,1732200503749 2024-11-21T14:48:24,927 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@25eb2ec 2024-11-21T14:48:24,927 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T14:48:24,929 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T14:48:24,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732200504925', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T14:48:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 2024-11-21T14:48:24,932 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T14:48:24,932 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:24,932 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732200504925" procId is: 4 2024-11-21T14:48:24,933 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T14:48:24,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:24,945 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c8157f840f749954d6795899b12eeca2, NAME => 'testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732200504925', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:24,956 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:24,956 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(1722): Closing c8157f840f749954d6795899b12eeca2, disabling compactions & flushes 2024-11-21T14:48:24,956 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:24,956 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:24,956 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. after waiting 0 ms 2024-11-21T14:48:24,956 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:24,957 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:24,957 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732200504925-pool-0 {}] regionserver.HRegion(1676): Region close journal for c8157f840f749954d6795899b12eeca2: Waiting for close lock at 1732200504956Disabling compacts and flushes for region at 1732200504956Disabling writes for close at 1732200504956Writing region close event to WAL at 1732200504956Closed at 1732200504956 2024-11-21T14:48:24,958 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T14:48:24,958 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732200504958"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732200504958"}]},"ts":"1732200504958"} 2024-11-21T14:48:24,961 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T14:48:24,962 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T14:48:24,962 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732200504925","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732200504962"}]},"ts":"1732200504962"} 2024-11-21T14:48:24,964 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732200504925, state=ENABLING in hbase:meta 2024-11-21T14:48:24,965 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {f7796871bd2a=0} racks are {/default-rack=0} 2024-11-21T14:48:24,965 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T14:48:24,965 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T14:48:24,965 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T14:48:24,966 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T14:48:24,966 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T14:48:24,966 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T14:48:24,966 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T14:48:24,966 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN}] 2024-11-21T14:48:24,967 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN 2024-11-21T14:48:24,968 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN; state=OFFLINE, location=f7796871bd2a,40095,1732200503824; forceNewPlan=false, retain=false 2024-11-21T14:48:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:25,119 INFO [f7796871bd2a:42929 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-21T14:48:25,120 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c8157f840f749954d6795899b12eeca2, regionState=OPENING, regionLocation=f7796871bd2a,40095,1732200503824 2024-11-21T14:48:25,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN because future has completed 2024-11-21T14:48:25,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8157f840f749954d6795899b12eeca2, server=f7796871bd2a,40095,1732200503824}] 2024-11-21T14:48:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:25,279 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T14:48:25,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37107, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T14:48:25,284 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:25,284 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c8157f840f749954d6795899b12eeca2, NAME => 'testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:25,284 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732200504925 c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,285 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:25,285 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,285 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,286 INFO [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,291 INFO [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8157f840f749954d6795899b12eeca2 columnFamilyName family 2024-11-21T14:48:25,291 DEBUG [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:25,291 INFO [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] regionserver.HStore(327): Store=c8157f840f749954d6795899b12eeca2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:25,291 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,292 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,292 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,292 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,292 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,293 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,300 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T14:48:25,301 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c8157f840f749954d6795899b12eeca2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70007876, jitterRate=0.04319864511489868}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T14:48:25,301 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:25,302 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c8157f840f749954d6795899b12eeca2: Running coprocessor pre-open hook at 1732200505285Writing region info on filesystem at 1732200505285Initializing all the Stores at 1732200505285Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200505285Cleaning up temporary data from old regions at 1732200505292 (+7 ms)Running coprocessor post-open hooks at 1732200505301 (+9 ms)Region opened successfully at 1732200505302 (+1 ms) 2024-11-21T14:48:25,302 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2., pid=6, masterSystemTime=1732200505279 2024-11-21T14:48:25,305 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:25,305 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:25,306 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c8157f840f749954d6795899b12eeca2, regionState=OPEN, openSeqNum=2, regionLocation=f7796871bd2a,40095,1732200503824 2024-11-21T14:48:25,309 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8157f840f749954d6795899b12eeca2, server=f7796871bd2a,40095,1732200503824 because future has completed 2024-11-21T14:48:25,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T14:48:25,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c8157f840f749954d6795899b12eeca2, server=f7796871bd2a,40095,1732200503824 in 184 msec 2024-11-21T14:48:25,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T14:48:25,314 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN in 346 msec 2024-11-21T14:48:25,315 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T14:48:25,315 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732200504925","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732200505315"}]},"ts":"1732200505315"} 2024-11-21T14:48:25,317 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732200504925, state=ENABLED in hbase:meta 2024-11-21T14:48:25,318 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T14:48:25,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732200504925 in 388 msec 2024-11-21T14:48:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42929 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T14:48:25,560 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732200504925 completed 2024-11-21T14:48:25,560 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732200504925 get assigned. Timeout = 60000ms 2024-11-21T14:48:25,561 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T14:48:25,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732200504925 assigned to meta. Checking AM states. 2024-11-21T14:48:25,564 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T14:48:25,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732200504925 assigned. 2024-11-21T14:48:25,568 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732200504925', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2., hostname=f7796871bd2a,40095,1732200503824, seqNum=2] 2024-11-21T14:48:25,568 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T14:48:25,570 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T14:48:25,574 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c8157f840f749954d6795899b12eeca2 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-21T14:48:25,584 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 365e17f5130f481f9e4844158d399dae 2024-11-21T14:48:25,599 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/.tmp/family/365e17f5130f481f9e4844158d399dae is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-21T14:48:25,600 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/.tmp/family/365e17f5130f481f9e4844158d399dae 2024-11-21T14:48:25,604 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/.tmp/family/365e17f5130f481f9e4844158d399dae as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae 2024-11-21T14:48:25,606 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae, delay=907 ms 2024-11-21T14:48:25,608 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae, entries=2, sequenceid=6, filesize=4.9 K 2024-11-21T14:48:25,609 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for c8157f840f749954d6795899b12eeca2 in 35ms, sequenceid=6, compaction requested=false 2024-11-21T14:48:25,609 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732200504925' 2024-11-21T14:48:25,610 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c8157f840f749954d6795899b12eeca2: 2024-11-21T14:48:25,953 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T14:48:26,514 DEBUG [hfile-prefetch-1732200503178 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 365e17f5130f481f9e4844158d399dae_0. 2024-11-21T14:48:26,515 DEBUG [hfile-prefetch-1732200503178 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 365e17f5130f481f9e4844158d399dae_117. 2024-11-21T14:48:26,515 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae, with total blocks 2, and data blocks 1 2024-11-21T14:48:26,515 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-21T14:48:26,515 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-21T14:48:26,516 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae. Will sleep 100ms and try the verification again. 2024-11-21T14:48:26,538 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-21T14:48:26,538 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-21T14:48:26,538 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:26,538 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T14:48:26,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-21T14:48:26,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732200506538 2024-11-21T14:48:26,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-21T14:48:26,611 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;f7796871bd2a:40095,5,FailOnTimeoutGroup] 2024-11-21T14:48:26,611 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7796871bd2a,40095,1732200503824' ***** 2024-11-21T14:48:26,611 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-21T14:48:26,611 INFO [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T14:48:26,612 INFO [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence 2024-11-21T14:48:26,612 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T14:48:26,612 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T14:48:26,612 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T14:48:26,612 INFO [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T14:48:26,612 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T14:48:26,616 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae, with total blocks 2, and data blocks 1 2024-11-21T14:48:26,616 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-21T14:48:26,616 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2293): found block for file 365e17f5130f481f9e4844158d399dae in the backing map. Acquiring read lock for offset 0 2024-11-21T14:48:26,616 DEBUG [hfile-prefetch-1732200503178 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae has now been fully cached. 2024-11-21T14:48:26,616 DEBUG [hfile-prefetch-1732200503178 {}] hfile.PrefetchExecutor(121): Prefetch completed for 365e17f5130f481f9e4844158d399dae 2024-11-21T14:48:26,622 DEBUG [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1487): Thread RS:0;f7796871bd2a:40095 started persisting bucket cache to file 2024-11-21T14:48:26,622 DEBUG [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:26,622 DEBUG [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-21T14:48:26,626 DEBUG [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-21T14:48:26,626 DEBUG [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732200506622 2024-11-21T14:48:26,626 DEBUG [RS:0;f7796871bd2a:40095 {}] bucket.BucketCache(1509): Thread RS:0;f7796871bd2a:40095 finished persisting bucket cache to file, renaming 2024-11-21T14:48:26,627 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T14:48:26,627 INFO [RS:0;f7796871bd2a:40095 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T14:48:26,627 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T14:48:26,627 INFO [RS:0;f7796871bd2a:40095 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T14:48:26,627 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(3091): Received CLOSE for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:26,627 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(959): stopping server f7796871bd2a,40095,1732200503824 2024-11-21T14:48:26,627 INFO [RS:0;f7796871bd2a:40095 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T14:48:26,627 INFO [RS:0;f7796871bd2a:40095 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;f7796871bd2a:40095. 2024-11-21T14:48:26,627 DEBUG [RS:0;f7796871bd2a:40095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:26,627 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T14:48:26,627 DEBUG [RS:0;f7796871bd2a:40095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:26,628 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c8157f840f749954d6795899b12eeca2, disabling compactions & flushes 2024-11-21T14:48:26,628 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T14:48:26,628 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1325): Online Regions={c8157f840f749954d6795899b12eeca2=testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.} 2024-11-21T14:48:26,628 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:26,628 DEBUG [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1351): Waiting on c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:26,628 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:26,628 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. after waiting 0 ms 2024-11-21T14:48:26,628 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:26,634 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-21T14:48:26,634 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:26,635 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c8157f840f749954d6795899b12eeca2: Waiting for close lock at 1732200506627Running coprocessor pre-close hooks at 1732200506627Disabling compacts and flushes for region at 1732200506628 (+1 ms)Disabling writes for close at 1732200506628Writing region close event to WAL at 1732200506629 (+1 ms)Running coprocessor post-close hooks at 1732200506634 (+5 ms)Closed at 1732200506634 2024-11-21T14:48:26,635 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:26,665 INFO [regionserver/f7796871bd2a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:26,828 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(976): stopping server f7796871bd2a,40095,1732200503824; all regions closed. 2024-11-21T14:48:26,830 DEBUG [RS:0;f7796871bd2a:40095 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs 2024-11-21T14:48:26,830 INFO [RS:0;f7796871bd2a:40095 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7796871bd2a%2C40095%2C1732200503824:(num 1732200504406) 2024-11-21T14:48:26,830 DEBUG [RS:0;f7796871bd2a:40095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:26,830 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:26,830 INFO [RS:0;f7796871bd2a:40095 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T14:48:26,831 INFO [RS:0;f7796871bd2a:40095 {}] hbase.ChoreService(370): Chore service for: regionserver/f7796871bd2a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T14:48:26,831 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T14:48:26,831 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T14:48:26,831 INFO [regionserver/f7796871bd2a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T14:48:26,831 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T14:48:26,831 INFO [RS:0;f7796871bd2a:40095 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T14:48:26,831 INFO [RS:0;f7796871bd2a:40095 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40095 2024-11-21T14:48:26,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7796871bd2a,40095,1732200503824 2024-11-21T14:48:26,835 INFO [RS:0;f7796871bd2a:40095 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T14:48:26,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:26,837 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7796871bd2a,40095,1732200503824] 2024-11-21T14:48:26,838 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7796871bd2a,40095,1732200503824 already deleted, retry=false 2024-11-21T14:48:26,838 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of f7796871bd2a,40095,1732200503824 on f7796871bd2a,42929,1732200503749 2024-11-21T14:48:26,843 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure f7796871bd2a,40095,1732200503824, splitWal=true, meta=false 2024-11-21T14:48:26,845 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for f7796871bd2a,40095,1732200503824 (carryingMeta=false) f7796871bd2a,40095,1732200503824/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@ca12b5[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T14:48:26,845 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure f7796871bd2a,40095,1732200503824, splitWal=true, meta=false 2024-11-21T14:48:26,846 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): f7796871bd2a,40095,1732200503824 had 1 regions 2024-11-21T14:48:26,847 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure f7796871bd2a,40095,1732200503824, splitWal=true, meta=false, isMeta: false 2024-11-21T14:48:26,847 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,40095,1732200503824-splitting 2024-11-21T14:48:26,847 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,40095,1732200503824-splitting dir is empty, no logs to split. 2024-11-21T14:48:26,847 INFO [PEWorker-2 {}] master.SplitWALManager(105): f7796871bd2a,40095,1732200503824 WAL count=0, meta=false 2024-11-21T14:48:26,849 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,40095,1732200503824-splitting dir is empty, no logs to split. 2024-11-21T14:48:26,849 INFO [PEWorker-2 {}] master.SplitWALManager(105): f7796871bd2a,40095,1732200503824 WAL count=0, meta=false 2024-11-21T14:48:26,849 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if f7796871bd2a,40095,1732200503824 WAL splitting is done? wals=0, meta=false 2024-11-21T14:48:26,850 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for f7796871bd2a,40095,1732200503824 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,40095,1732200503824-splitting does not exist 2024-11-21T14:48:26,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN}] 2024-11-21T14:48:26,852 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN 2024-11-21T14:48:26,853 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T14:48:26,937 INFO [RS:0;f7796871bd2a:40095 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T14:48:26,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:26,937 INFO [RS:0;f7796871bd2a:40095 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7796871bd2a,40095,1732200503824; zookeeper connection closed. 2024-11-21T14:48:26,937 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40095-0x1001a8dd2950001, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:26,938 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@48b34ad9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@48b34ad9 2024-11-21T14:48:27,003 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c8157f840f749954d6795899b12eeca2, regionState=OPENING, regionLocation=f7796871bd2a,46135,1732200503872 2024-11-21T14:48:27,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN because future has completed 2024-11-21T14:48:27,007 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8157f840f749954d6795899b12eeca2, server=f7796871bd2a,46135,1732200503872}] 2024-11-21T14:48:27,128 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/f7796871bd2a:0 server-side Connection retries=45 2024-11-21T14:48:27,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:27,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:27,128 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T14:48:27,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T14:48:27,128 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T14:48:27,128 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T14:48:27,129 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T14:48:27,129 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38029 2024-11-21T14:48:27,131 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38029 connecting to ZooKeeper ensemble=127.0.0.1:65150 2024-11-21T14:48:27,131 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:27,131 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:27,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:380290x0, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T14:48:27,136 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38029-0x1001a8dd2950003 connected 2024-11-21T14:48:27,136 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-21T14:48:27,136 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-21T14:48:27,136 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T14:48:27,137 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.cache 2024-11-21T14:48:27,137 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-21T14:48:27,137 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-21T14:48:27,137 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-21T14:48:27,138 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-21T14:48:27,139 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-21T14:48:27,140 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-21T14:48:27,140 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T14:48:27,140 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T14:48:27,141 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-21T14:48:27,142 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T14:48:27,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38029 2024-11-21T14:48:27,144 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38029 2024-11-21T14:48:27,148 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-21T14:48:27,148 DEBUG [Thread-201 {}] bucket.BucketCache(1584): Cached File Entry:<365e17f5130f481f9e4844158d399dae,> 2024-11-21T14:48:27,148 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38029 2024-11-21T14:48:27,149 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-21T14:48:27,149 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-21T14:48:27,149 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence is complete. 2024-11-21T14:48:27,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38029 2024-11-21T14:48:27,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38029 2024-11-21T14:48:27,155 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(746): ClusterId : 46221a4c-d985-41d4-8427-7addb6ced2e6 2024-11-21T14:48:27,155 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T14:48:27,157 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T14:48:27,157 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T14:48:27,159 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T14:48:27,160 DEBUG [RS:2;f7796871bd2a:38029 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c7e7f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=f7796871bd2a/172.17.0.2:0 2024-11-21T14:48:27,161 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,161 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c8157f840f749954d6795899b12eeca2, NAME => 'testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.', STARTKEY => '', ENDKEY => ''} 2024-11-21T14:48:27,162 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732200504925 c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,162 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T14:48:27,162 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,162 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,163 INFO [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,164 INFO [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c8157f840f749954d6795899b12eeca2 columnFamilyName family 2024-11-21T14:48:27,164 DEBUG [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T14:48:27,166 DEBUG [StoreFileOpener-c8157f840f749954d6795899b12eeca2-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae, delay=972 ms 2024-11-21T14:48:27,168 DEBUG [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae 2024-11-21T14:48:27,168 INFO [StoreOpener-c8157f840f749954d6795899b12eeca2-1 {}] regionserver.HStore(327): Store=c8157f840f749954d6795899b12eeca2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T14:48:27,168 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,169 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,170 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,170 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,170 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,172 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,173 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c8157f840f749954d6795899b12eeca2; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58822145, jitterRate=-0.12348173558712006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T14:48:27,173 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,174 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c8157f840f749954d6795899b12eeca2: Running coprocessor pre-open hook at 1732200507162Writing region info on filesystem at 1732200507162Initializing all the Stores at 1732200507163 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732200507163Cleaning up temporary data from old regions at 1732200507170 (+7 ms)Running coprocessor post-open hooks at 1732200507173 (+3 ms)Region opened successfully at 1732200507174 (+1 ms) 2024-11-21T14:48:27,175 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2., pid=9, masterSystemTime=1732200507159 2024-11-21T14:48:27,176 DEBUG [RS:2;f7796871bd2a:38029 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;f7796871bd2a:38029 2024-11-21T14:48:27,176 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T14:48:27,176 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T14:48:27,176 DEBUG [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T14:48:27,177 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(2659): reportForDuty to master=f7796871bd2a,42929,1732200503749 with port=38029, startcode=1732200507127 2024-11-21T14:48:27,177 DEBUG [RS:2;f7796871bd2a:38029 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T14:48:27,178 DEBUG [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,178 INFO [RS_OPEN_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,178 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c8157f840f749954d6795899b12eeca2, regionState=OPEN, openSeqNum=10, regionLocation=f7796871bd2a,46135,1732200503872 2024-11-21T14:48:27,180 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32837, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T14:48:27,180 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(363): Checking decommissioned status of RegionServer f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,181 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42929 {}] master.ServerManager(517): Registering regionserver=f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c8157f840f749954d6795899b12eeca2, server=f7796871bd2a,46135,1732200503872 because future has completed 2024-11-21T14:48:27,182 DEBUG [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174 2024-11-21T14:48:27,182 DEBUG [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-21T14:48:27,182 DEBUG [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T14:48:27,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-21T14:48:27,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c8157f840f749954d6795899b12eeca2, server=f7796871bd2a,46135,1732200503872 in 176 msec 2024-11-21T14:48:27,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:27,185 DEBUG [RS:2;f7796871bd2a:38029 {}] zookeeper.ZKUtil(111): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,185 WARN [RS:2;f7796871bd2a:38029 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T14:48:27,185 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [f7796871bd2a,38029,1732200507127] 2024-11-21T14:48:27,185 INFO [RS:2;f7796871bd2a:38029 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T14:48:27,185 DEBUG [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T14:48:27,186 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732200504925, region=c8157f840f749954d6795899b12eeca2, ASSIGN in 333 msec 2024-11-21T14:48:27,186 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server f7796871bd2a,40095,1732200503824 after splitting done 2024-11-21T14:48:27,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure f7796871bd2a,40095,1732200503824, splitWal=true, meta=false in 347 msec 2024-11-21T14:48:27,191 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T14:48:27,194 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T14:48:27,194 INFO [RS:2;f7796871bd2a:38029 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T14:48:27,194 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,195 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T14:48:27,195 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T14:48:27,195 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/f7796871bd2a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/f7796871bd2a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T14:48:27,196 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:27,197 DEBUG [RS:2;f7796871bd2a:38029 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/f7796871bd2a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T14:48:27,199 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,199 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,199 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,199 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,199 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,199 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,38029,1732200507127-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T14:48:27,220 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T14:48:27,220 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=f7796871bd2a,38029,1732200507127-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,220 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,220 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.Replication(171): f7796871bd2a,38029,1732200507127 started 2024-11-21T14:48:27,237 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T14:48:27,237 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(1482): Serving as f7796871bd2a,38029,1732200507127, RpcServer on f7796871bd2a/172.17.0.2:38029, sessionid=0x1001a8dd2950003 2024-11-21T14:48:27,238 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T14:48:27,238 DEBUG [RS:2;f7796871bd2a:38029 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,238 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,38029,1732200507127' 2024-11-21T14:48:27,238 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T14:48:27,238 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T14:48:27,239 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T14:48:27,239 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T14:48:27,239 DEBUG [RS:2;f7796871bd2a:38029 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,239 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'f7796871bd2a,38029,1732200507127' 2024-11-21T14:48:27,239 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T14:48:27,240 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T14:48:27,240 DEBUG [RS:2;f7796871bd2a:38029 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T14:48:27,240 INFO [RS:2;f7796871bd2a:38029 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T14:48:27,240 INFO [RS:2;f7796871bd2a:38029 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T14:48:27,341 INFO [RS:2;f7796871bd2a:38029 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T14:48:27,342 INFO [RS:2;f7796871bd2a:38029 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=f7796871bd2a%2C38029%2C1732200507127, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,38029,1732200507127, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs, maxLogs=55 2024-11-21T14:48:27,361 INFO [RS:2;f7796871bd2a:38029 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/WALs/f7796871bd2a,38029,1732200507127/f7796871bd2a%2C38029%2C1732200507127.1732200507343 2024-11-21T14:48:27,362 DEBUG [RS:2;f7796871bd2a:38029 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-21T14:48:27,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T14:48:27,738 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T14:48:27,738 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:27,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:27,738 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:27,738 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T14:48:27,738 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T14:48:27,739 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1788314048, stopped=false 2024-11-21T14:48:27,739 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=f7796871bd2a,42929,1732200503749 2024-11-21T14:48:27,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:27,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:27,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T14:48:27,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:27,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T14:48:27,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:27,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:27,741 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T14:48:27,741 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:27,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:27,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:27,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:27,742 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7796871bd2a,46135,1732200503872' ***** 2024-11-21T14:48:27,742 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T14:48:27,742 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'f7796871bd2a,38029,1732200507127' ***** 2024-11-21T14:48:27,742 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T14:48:27,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T14:48:27,742 INFO [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T14:48:27,742 INFO [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence 2024-11-21T14:48:27,742 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T14:48:27,742 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T14:48:27,743 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T14:48:27,743 INFO [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T14:48:27,743 INFO [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1738): Disabling cache 2024-11-21T14:48:27,743 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T14:48:27,743 INFO [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2a2acfc1-c8b8-cd27-b750-847ea1786fac/bucket.persistence 2024-11-21T14:48:27,743 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-21T14:48:27,743 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-21T14:48:27,743 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-21T14:48:27,743 INFO [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-21T14:48:27,744 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-21T14:48:27,753 DEBUG [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1487): Thread RS:1;f7796871bd2a:46135 started persisting bucket cache to file 2024-11-21T14:48:27,753 DEBUG [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:27,753 DEBUG [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-21T14:48:27,754 DEBUG [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1487): Thread RS:2;f7796871bd2a:38029 started persisting bucket cache to file 2024-11-21T14:48:27,754 DEBUG [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-21T14:48:27,754 DEBUG [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-21T14:48:27,757 DEBUG [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-21T14:48:27,757 DEBUG [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732200507753 2024-11-21T14:48:27,757 DEBUG [RS:1;f7796871bd2a:46135 {}] bucket.BucketCache(1509): Thread RS:1;f7796871bd2a:46135 finished persisting bucket cache to file, renaming 2024-11-21T14:48:27,757 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T14:48:27,757 INFO [RS:1;f7796871bd2a:46135 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T14:48:27,757 INFO [RS:1;f7796871bd2a:46135 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T14:48:27,757 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T14:48:27,757 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(3091): Received CLOSE for c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,757 ERROR [RS:2;f7796871bd2a:38029 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T14:48:27,757 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(959): stopping server f7796871bd2a,46135,1732200503872 2024-11-21T14:48:27,757 INFO [RS:1;f7796871bd2a:46135 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T14:48:27,758 DEBUG [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-21T14:48:27,758 INFO [RS:1;f7796871bd2a:46135 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;f7796871bd2a:46135. 2024-11-21T14:48:27,758 DEBUG [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732200507754 2024-11-21T14:48:27,758 DEBUG [RS:1;f7796871bd2a:46135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:27,758 DEBUG [RS:2;f7796871bd2a:38029 {}] bucket.BucketCache(1509): Thread RS:2;f7796871bd2a:38029 finished persisting bucket cache to file, renaming 2024-11-21T14:48:27,758 DEBUG [RS:1;f7796871bd2a:46135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:27,758 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T14:48:27,758 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T14:48:27,758 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c8157f840f749954d6795899b12eeca2, disabling compactions & flushes 2024-11-21T14:48:27,758 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T14:48:27,758 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T14:48:27,758 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,758 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T14:48:27,758 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T14:48:27,758 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,758 INFO [RS:2;f7796871bd2a:38029 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T14:48:27,758 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. after waiting 0 ms 2024-11-21T14:48:27,758 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T14:48:27,758 INFO [RS:2;f7796871bd2a:38029 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T14:48:27,758 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,758 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(959): stopping server f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,758 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-21T14:48:27,758 INFO [RS:2;f7796871bd2a:38029 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T14:48:27,758 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, c8157f840f749954d6795899b12eeca2=testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2.} 2024-11-21T14:48:27,758 INFO [RS:2;f7796871bd2a:38029 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;f7796871bd2a:38029. 2024-11-21T14:48:27,758 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-21T14:48:27,759 DEBUG [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c8157f840f749954d6795899b12eeca2 2024-11-21T14:48:27,759 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T14:48:27,759 DEBUG [RS:2;f7796871bd2a:38029 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T14:48:27,759 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T14:48:27,759 DEBUG [RS:2;f7796871bd2a:38029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:27,759 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T14:48:27,759 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T14:48:27,759 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T14:48:27,759 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(976): stopping server f7796871bd2a,38029,1732200507127; all regions closed. 2024-11-21T14:48:27,759 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-21T14:48:27,759 DEBUG [StoreFileCloser-c8157f840f749954d6795899b12eeca2-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/family/365e17f5130f481f9e4844158d399dae 2024-11-21T14:48:27,761 DEBUG [RS:2;f7796871bd2a:38029 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs 2024-11-21T14:48:27,761 INFO [RS:2;f7796871bd2a:38029 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7796871bd2a%2C38029%2C1732200507127:(num 1732200507343) 2024-11-21T14:48:27,761 DEBUG [RS:2;f7796871bd2a:38029 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:27,761 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:27,761 INFO [RS:2;f7796871bd2a:38029 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T14:48:27,762 INFO [RS:2;f7796871bd2a:38029 {}] hbase.ChoreService(370): Chore service for: regionserver/f7796871bd2a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T14:48:27,762 INFO [regionserver/f7796871bd2a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T14:48:27,762 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T14:48:27,762 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T14:48:27,762 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T14:48:27,762 INFO [RS:2;f7796871bd2a:38029 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T14:48:27,762 INFO [RS:2;f7796871bd2a:38029 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38029 2024-11-21T14:48:27,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7796871bd2a,38029,1732200507127 2024-11-21T14:48:27,765 INFO [RS:2;f7796871bd2a:38029 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T14:48:27,765 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:27,766 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7796871bd2a,38029,1732200507127] 2024-11-21T14:48:27,766 INFO [regionserver/f7796871bd2a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:27,767 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7796871bd2a,38029,1732200507127 already deleted, retry=false 2024-11-21T14:48:27,767 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7796871bd2a,38029,1732200507127 expired; onlineServers=1 2024-11-21T14:48:27,770 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/default/testBlockEvictionOnGracefulStop1732200504925/c8157f840f749954d6795899b12eeca2/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-21T14:48:27,770 INFO [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,771 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c8157f840f749954d6795899b12eeca2: Waiting for close lock at 1732200507758Running coprocessor pre-close hooks at 1732200507758Disabling compacts and flushes for region at 1732200507758Disabling writes for close at 1732200507758Writing region close event to WAL at 1732200507760 (+2 ms)Running coprocessor post-close hooks at 1732200507770 (+10 ms)Closed at 1732200507770 2024-11-21T14:48:27,771 DEBUG [RS_CLOSE_REGION-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2. 2024-11-21T14:48:27,775 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 830d3807f0df45649f0f6fe741f9ff8a 2024-11-21T14:48:27,791 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/info/830d3807f0df45649f0f6fe741f9ff8a is 209, key is testBlockEvictionOnGracefulStop1732200504925,,1732200504929.c8157f840f749954d6795899b12eeca2./info:regioninfo/1732200507178/Put/seqid=0 2024-11-21T14:48:27,791 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/info/830d3807f0df45649f0f6fe741f9ff8a 2024-11-21T14:48:27,800 INFO [regionserver/f7796871bd2a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:27,801 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f193f7abec8644ad8e3abf18156ec37d 2024-11-21T14:48:27,815 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/ns/f193f7abec8644ad8e3abf18156ec37d is 43, key is default/ns:d/1732200504820/Put/seqid=0 2024-11-21T14:48:27,816 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/ns/f193f7abec8644ad8e3abf18156ec37d 2024-11-21T14:48:27,825 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ec7b2ba2bbc24768a2a49f39bd432443 2024-11-21T14:48:27,838 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/table/ec7b2ba2bbc24768a2a49f39bd432443 is 80, key is testBlockEvictionOnGracefulStop1732200504925/table:state/1732200505315/Put/seqid=0 2024-11-21T14:48:27,839 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/table/ec7b2ba2bbc24768a2a49f39bd432443 2024-11-21T14:48:27,842 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/info/830d3807f0df45649f0f6fe741f9ff8a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/830d3807f0df45649f0f6fe741f9ff8a 2024-11-21T14:48:27,843 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/830d3807f0df45649f0f6fe741f9ff8a, delay=968 ms 2024-11-21T14:48:27,844 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/830d3807f0df45649f0f6fe741f9ff8a, entries=14, sequenceid=13, filesize=7.5 K 2024-11-21T14:48:27,845 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/ns/f193f7abec8644ad8e3abf18156ec37d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f193f7abec8644ad8e3abf18156ec37d 2024-11-21T14:48:27,846 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f193f7abec8644ad8e3abf18156ec37d, delay=919 ms 2024-11-21T14:48:27,847 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f193f7abec8644ad8e3abf18156ec37d, entries=2, sequenceid=13, filesize=5.0 K 2024-11-21T14:48:27,847 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/.tmp/table/ec7b2ba2bbc24768a2a49f39bd432443 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/ec7b2ba2bbc24768a2a49f39bd432443 2024-11-21T14:48:27,848 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/ec7b2ba2bbc24768a2a49f39bd432443, delay=964 ms 2024-11-21T14:48:27,849 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/ec7b2ba2bbc24768a2a49f39bd432443, entries=2, sequenceid=13, filesize=5.3 K 2024-11-21T14:48:27,850 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=13, compaction requested=false 2024-11-21T14:48:27,851 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/info/830d3807f0df45649f0f6fe741f9ff8a 2024-11-21T14:48:27,851 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/ns/f193f7abec8644ad8e3abf18156ec37d 2024-11-21T14:48:27,852 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/table/ec7b2ba2bbc24768a2a49f39bd432443 2024-11-21T14:48:27,857 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-21T14:48:27,857 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T14:48:27,857 INFO [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T14:48:27,857 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732200507758Running coprocessor pre-close hooks at 1732200507759 (+1 ms)Disabling compacts and flushes for region at 1732200507759Disabling writes for close at 1732200507759Obtaining lock to block concurrent updates at 1732200507759Preparing flush snapshotting stores in 1588230740 at 1732200507759Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732200507760 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732200507760Flushing 1588230740/info: creating writer at 1732200507760Flushing 1588230740/info: appending metadata at 1732200507790 (+30 ms)Flushing 1588230740/info: closing flushed file at 1732200507790Flushing 1588230740/ns: creating writer at 1732200507795 (+5 ms)Flushing 1588230740/ns: appending metadata at 1732200507815 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732200507815Flushing 1588230740/table: creating writer at 1732200507818 (+3 ms)Flushing 1588230740/table: appending metadata at 1732200507838 (+20 ms)Flushing 1588230740/table: closing flushed file at 1732200507838Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63c9799a: reopening flushed file at 1732200507841 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b7eda21: reopening flushed file at 1732200507844 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@24c1b024: reopening flushed file at 1732200507847 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 91ms, sequenceid=13, compaction requested=false at 1732200507850 (+3 ms)Writing region close event to WAL at 1732200507852 (+2 ms)Running coprocessor post-close hooks at 1732200507857 (+5 ms)Closed at 1732200507857 2024-11-21T14:48:27,857 DEBUG [RS_CLOSE_META-regionserver/f7796871bd2a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T14:48:27,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:27,867 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38029-0x1001a8dd2950003, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:27,867 INFO [RS:2;f7796871bd2a:38029 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T14:48:27,867 INFO [RS:2;f7796871bd2a:38029 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7796871bd2a,38029,1732200507127; zookeeper connection closed. 2024-11-21T14:48:27,868 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@447c1286 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@447c1286 2024-11-21T14:48:27,959 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(976): stopping server f7796871bd2a,46135,1732200503872; all regions closed. 2024-11-21T14:48:27,961 DEBUG [RS:1;f7796871bd2a:46135 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs 2024-11-21T14:48:27,961 INFO [RS:1;f7796871bd2a:46135 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7796871bd2a%2C46135%2C1732200503872.meta:.meta(num 1732200504773) 2024-11-21T14:48:27,963 DEBUG [RS:1;f7796871bd2a:46135 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/oldWALs 2024-11-21T14:48:27,963 INFO [RS:1;f7796871bd2a:46135 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL f7796871bd2a%2C46135%2C1732200503872:(num 1732200504401) 2024-11-21T14:48:27,963 DEBUG [RS:1;f7796871bd2a:46135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T14:48:27,963 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T14:48:27,963 INFO [RS:1;f7796871bd2a:46135 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T14:48:27,963 INFO [RS:1;f7796871bd2a:46135 {}] hbase.ChoreService(370): Chore service for: regionserver/f7796871bd2a:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T14:48:27,963 INFO [RS:1;f7796871bd2a:46135 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T14:48:27,963 INFO [regionserver/f7796871bd2a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T14:48:27,964 INFO [RS:1;f7796871bd2a:46135 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46135 2024-11-21T14:48:27,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/f7796871bd2a,46135,1732200503872 2024-11-21T14:48:27,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T14:48:27,966 INFO [RS:1;f7796871bd2a:46135 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T14:48:27,968 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [f7796871bd2a,46135,1732200503872] 2024-11-21T14:48:27,969 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/f7796871bd2a,46135,1732200503872 already deleted, retry=false 2024-11-21T14:48:27,969 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; f7796871bd2a,46135,1732200503872 expired; onlineServers=0 2024-11-21T14:48:27,969 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'f7796871bd2a,42929,1732200503749' ***** 2024-11-21T14:48:27,969 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T14:48:27,969 INFO [M:0;f7796871bd2a:42929 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T14:48:27,969 INFO [M:0;f7796871bd2a:42929 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T14:48:27,970 DEBUG [M:0;f7796871bd2a:42929 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T14:48:27,970 DEBUG [M:0;f7796871bd2a:42929 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T14:48:27,970 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T14:48:27,970 DEBUG [master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.large.0-1732200504152 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.large.0-1732200504152,5,FailOnTimeoutGroup] 2024-11-21T14:48:27,970 DEBUG [master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.small.0-1732200504152 {}] cleaner.HFileCleaner(306): Exit Thread[master/f7796871bd2a:0:becomeActiveMaster-HFileCleaner.small.0-1732200504152,5,FailOnTimeoutGroup] 2024-11-21T14:48:27,970 INFO [M:0;f7796871bd2a:42929 {}] hbase.ChoreService(370): Chore service for: master/f7796871bd2a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T14:48:27,970 INFO [M:0;f7796871bd2a:42929 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T14:48:27,970 DEBUG [M:0;f7796871bd2a:42929 {}] master.HMaster(1795): Stopping service threads 2024-11-21T14:48:27,970 INFO [M:0;f7796871bd2a:42929 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T14:48:27,970 INFO [M:0;f7796871bd2a:42929 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T14:48:27,971 INFO [M:0;f7796871bd2a:42929 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T14:48:27,971 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T14:48:27,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T14:48:27,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T14:48:27,972 DEBUG [M:0;f7796871bd2a:42929 {}] zookeeper.ZKUtil(347): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T14:48:27,972 WARN [M:0;f7796871bd2a:42929 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T14:48:27,972 INFO [M:0;f7796871bd2a:42929 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/.lastflushedseqids 2024-11-21T14:48:27,978 INFO [M:0;f7796871bd2a:42929 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T14:48:27,979 INFO [M:0;f7796871bd2a:42929 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T14:48:27,979 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T14:48:27,979 INFO [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:27,979 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:27,979 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T14:48:27,979 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:27,979 INFO [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-21T14:48:27,989 DEBUG [M:0;f7796871bd2a:42929 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d49e86550eef4e3296ac56008e545308 2024-11-21T14:48:28,003 DEBUG [M:0;f7796871bd2a:42929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d49e86550eef4e3296ac56008e545308 is 82, key is hbase:meta,,1/info:regioninfo/1732200504806/Put/seqid=0 2024-11-21T14:48:28,004 INFO [M:0;f7796871bd2a:42929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d49e86550eef4e3296ac56008e545308 2024-11-21T14:48:28,012 DEBUG [M:0;f7796871bd2a:42929 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3bcf169b6d6c41ab8f8269b31ede4ac7 2024-11-21T14:48:28,025 DEBUG [M:0;f7796871bd2a:42929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bcf169b6d6c41ab8f8269b31ede4ac7 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732200505319/Put/seqid=0 2024-11-21T14:48:28,026 INFO [M:0;f7796871bd2a:42929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bcf169b6d6c41ab8f8269b31ede4ac7 2024-11-21T14:48:28,034 DEBUG [M:0;f7796871bd2a:42929 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 33bd2fafae324cd9ae8cf16c845f0510 2024-11-21T14:48:28,049 DEBUG [M:0;f7796871bd2a:42929 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33bd2fafae324cd9ae8cf16c845f0510 is 69, key is f7796871bd2a,38029,1732200507127/rs:state/1732200507181/Put/seqid=0 2024-11-21T14:48:28,049 INFO [M:0;f7796871bd2a:42929 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33bd2fafae324cd9ae8cf16c845f0510 2024-11-21T14:48:28,051 INFO [M:0;f7796871bd2a:42929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 33bd2fafae324cd9ae8cf16c845f0510 2024-11-21T14:48:28,052 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d49e86550eef4e3296ac56008e545308 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d49e86550eef4e3296ac56008e545308 2024-11-21T14:48:28,053 INFO [M:0;f7796871bd2a:42929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d49e86550eef4e3296ac56008e545308, entries=8, sequenceid=83, filesize=5.5 K 2024-11-21T14:48:28,054 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bcf169b6d6c41ab8f8269b31ede4ac7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bcf169b6d6c41ab8f8269b31ede4ac7 2024-11-21T14:48:28,055 INFO [M:0;f7796871bd2a:42929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bcf169b6d6c41ab8f8269b31ede4ac7, entries=9, sequenceid=83, filesize=6.4 K 2024-11-21T14:48:28,056 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/33bd2fafae324cd9ae8cf16c845f0510 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33bd2fafae324cd9ae8cf16c845f0510 2024-11-21T14:48:28,057 INFO [M:0;f7796871bd2a:42929 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 33bd2fafae324cd9ae8cf16c845f0510 2024-11-21T14:48:28,057 INFO [M:0;f7796871bd2a:42929 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/02c3536f-89f0-65f7-1b75-633ffc930d26/b118454f-de69-c6da-3536-22dbd23c0174/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/33bd2fafae324cd9ae8cf16c845f0510, entries=3, sequenceid=83, filesize=5.3 K 2024-11-21T14:48:28,058 INFO [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false 2024-11-21T14:48:28,060 INFO [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T14:48:28,060 DEBUG [M:0;f7796871bd2a:42929 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732200507979Disabling compacts and flushes for region at 1732200507979Disabling writes for close at 1732200507979Obtaining lock to block concurrent updates at 1732200507979Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732200507979Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732200507980 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732200507980Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732200507980Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732200508003 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732200508003Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732200508005 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732200508025 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732200508025Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732200508028 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732200508048 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732200508048Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77d05bfd: reopening flushed file at 1732200508051 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a776f4d: reopening flushed file at 1732200508053 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@375eff05: reopening flushed file at 1732200508055 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=83, compaction requested=false at 1732200508058 (+3 ms)Writing region close event to WAL at 1732200508060 (+2 ms)Closed at 1732200508060 2024-11-21T14:48:28,061 INFO [M:0;f7796871bd2a:42929 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T14:48:28,061 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T14:48:28,061 INFO [M:0;f7796871bd2a:42929 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42929 2024-11-21T14:48:28,061 INFO [M:0;f7796871bd2a:42929 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T14:48:28,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:28,068 INFO [RS:1;f7796871bd2a:46135 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T14:48:28,068 INFO [RS:1;f7796871bd2a:46135 {}] regionserver.HRegionServer(1031): Exiting; stopping=f7796871bd2a,46135,1732200503872; zookeeper connection closed. 2024-11-21T14:48:28,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46135-0x1001a8dd2950002, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:28,068 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@103b9ba2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@103b9ba2 2024-11-21T14:48:28,069 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T14:48:28,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:28,163 INFO [M:0;f7796871bd2a:42929 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T14:48:28,163 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42929-0x1001a8dd2950000, quorum=127.0.0.1:65150, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T14:48:28,167 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T14:48:28,169 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T14:48:28,169 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T14:48:28,177 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=204 (was 196) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3333 (was 3425)