2024-11-27 05:45:10,279 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-27 05:45:10,312 main DEBUG Took 0.030898 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 05:45:10,313 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 05:45:10,313 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 05:45:10,314 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 05:45:10,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,332 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 05:45:10,357 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,359 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,361 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,362 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,363 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,365 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,367 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,368 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,368 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,369 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,369 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,370 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,370 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,371 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,371 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,372 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,372 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 05:45:10,373 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,373 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 05:45:10,375 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 05:45:10,377 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 05:45:10,379 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 05:45:10,380 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 05:45:10,381 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 05:45:10,382 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 05:45:10,393 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 05:45:10,396 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 05:45:10,397 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 05:45:10,398 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 05:45:10,398 main DEBUG createAppenders(={Console}) 2024-11-27 05:45:10,399 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-27 05:45:10,399 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-27 05:45:10,400 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-27 05:45:10,400 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 05:45:10,401 main DEBUG OutputStream closed 2024-11-27 05:45:10,401 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 05:45:10,401 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 05:45:10,401 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-27 05:45:10,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 05:45:10,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 05:45:10,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 05:45:10,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 05:45:10,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 05:45:10,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 05:45:10,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 05:45:10,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 05:45:10,512 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 05:45:10,513 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 05:45:10,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 05:45:10,514 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 05:45:10,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 05:45:10,515 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 05:45:10,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 05:45:10,516 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 05:45:10,517 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 05:45:10,518 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 05:45:10,521 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 05:45:10,522 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-27 05:45:10,522 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 05:45:10,524 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-27T05:45:10,543 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-27 05:45:10,546 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 05:45:10,546 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T05:45:10,838 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666 2024-11-27T05:45:10,864 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=300, ProcessCount=11, AvailableMemoryMB=8499 2024-11-27T05:45:10,980 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T05:45:11,104 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:11,140 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/cluster_c270135b-89d8-5272-dc87-c83a2253bf11, deleteOnExit=true 2024-11-27T05:45:11,248 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/cluster_c270135b-89d8-5272-dc87-c83a2253bf11/zookeeper_0, clientPort=54797, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/cluster_c270135b-89d8-5272-dc87-c83a2253bf11/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/cluster_c270135b-89d8-5272-dc87-c83a2253bf11/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T05:45:11,261 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54797 2024-11-27T05:45:11,262 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:11,356 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 with version=8 2024-11-27T05:45:11,357 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/hbase-staging 2024-11-27T05:45:11,477 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T05:45:11,713 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4da2d47fc9ac:0 server-side Connection retries=45 2024-11-27T05:45:11,723 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:11,725 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:11,733 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T05:45:11,733 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:11,734 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T05:45:11,874 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T05:45:11,955 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T05:45:11,995 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T05:45:12,006 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T05:45:12,009 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T05:45:12,048 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 13250 (auto-detected) 2024-11-27T05:45:12,050 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T05:45:12,073 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40489 2024-11-27T05:45:12,096 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40489 connecting to ZooKeeper ensemble=127.0.0.1:54797 2024-11-27T05:45:12,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:404890x0, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T05:45:12,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40489-0x1017ad2eae10000 connected 2024-11-27T05:45:12,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:12,367 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:12,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:12,382 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93, hbase.cluster.distributed=false 2024-11-27T05:45:12,421 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T05:45:12,442 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40489 2024-11-27T05:45:12,443 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40489 2024-11-27T05:45:12,444 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40489 2024-11-27T05:45:12,449 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40489 2024-11-27T05:45:12,450 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40489 2024-11-27T05:45:12,586 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4da2d47fc9ac:0 server-side Connection retries=45 2024-11-27T05:45:12,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:12,589 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:12,590 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T05:45:12,590 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:12,591 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T05:45:12,595 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T05:45:12,600 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T05:45:12,602 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44713 2024-11-27T05:45:12,605 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44713 connecting to ZooKeeper ensemble=127.0.0.1:54797 2024-11-27T05:45:12,606 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:12,607 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:12,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:447130x0, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T05:45:12,626 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:447130x0, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:12,627 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44713-0x1017ad2eae10001 connected 2024-11-27T05:45:12,635 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T05:45:12,653 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.cache 2024-11-27T05:45:12,661 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T05:45:12,663 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T05:45:12,666 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T05:45:12,670 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T05:45:12,670 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T05:45:12,682 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T05:45:12,683 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.persistence is complete. 2024-11-27T05:45:12,690 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T05:45:12,713 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T05:45:12,717 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T05:45:12,725 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T05:45:12,736 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44713 2024-11-27T05:45:12,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44713 2024-11-27T05:45:12,743 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44713 2024-11-27T05:45:12,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44713 2024-11-27T05:45:12,745 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44713 2024-11-27T05:45:12,776 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4da2d47fc9ac:0 server-side Connection retries=45 2024-11-27T05:45:12,776 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:12,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:12,777 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T05:45:12,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:12,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T05:45:12,778 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T05:45:12,778 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T05:45:12,780 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37167 2024-11-27T05:45:12,783 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37167 connecting to ZooKeeper ensemble=127.0.0.1:54797 2024-11-27T05:45:12,784 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:12,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:12,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371670x0, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T05:45:12,846 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:371670x0, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:12,847 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T05:45:12,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37167-0x1017ad2eae10002 connected 2024-11-27T05:45:12,872 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.cache 2024-11-27T05:45:12,872 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T05:45:12,873 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T05:45:12,873 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T05:45:12,880 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T05:45:12,881 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T05:45:12,883 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T05:45:12,883 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.persistence is complete. 2024-11-27T05:45:12,891 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T05:45:12,896 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T05:45:12,899 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T05:45:12,902 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T05:45:12,937 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37167 2024-11-27T05:45:12,938 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37167 2024-11-27T05:45:12,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37167 2024-11-27T05:45:12,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37167 2024-11-27T05:45:12,953 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37167 2024-11-27T05:45:12,974 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4da2d47fc9ac:40489 2024-11-27T05:45:12,976 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:12,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:12,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:12,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:13,006 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:13,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:13,054 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T05:45:13,056 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4da2d47fc9ac,40489,1732686311548 from backup master directory 2024-11-27T05:45:13,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:13,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:13,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:13,067 WARN [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T05:45:13,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:13,067 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:13,070 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T05:45:13,072 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T05:45:13,084 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/hbase.id] with ID: 678c977a-db27-498d-a76e-34e244be1efe 2024-11-27T05:45:13,084 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.tmp/hbase.id 2024-11-27T05:45:13,104 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/hbase.id] 2024-11-27T05:45:13,199 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:13,208 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T05:45:13,236 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 25ms. 2024-11-27T05:45:13,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:13,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:13,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:13,303 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T05:45:13,308 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T05:45:13,384 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T05:45:13,392 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:13,502 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store 2024-11-27T05:45:13,547 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T05:45:13,552 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:13,554 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T05:45:13,554 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:13,555 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:13,557 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T05:45:13,558 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:13,558 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:13,559 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732686313554Disabling compacts and flushes for region at 1732686313554Disabling writes for close at 1732686313557 (+3 ms)Writing region close event to WAL at 1732686313558 (+1 ms)Closed at 1732686313558 2024-11-27T05:45:13,559 WARN [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/.initializing 2024-11-27T05:45:13,560 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/WALs/4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:13,595 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T05:45:13,612 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4da2d47fc9ac%2C40489%2C1732686311548, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/WALs/4da2d47fc9ac,40489,1732686311548, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/oldWALs, maxLogs=10 2024-11-27T05:45:13,673 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/WALs/4da2d47fc9ac,40489,1732686311548/4da2d47fc9ac%2C40489%2C1732686311548.1732686313619 2024-11-27T05:45:13,674 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:13,675 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:13,676 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:13,682 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,684 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,774 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T05:45:13,780 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:13,782 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:13,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T05:45:13,795 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:13,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:13,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,806 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T05:45:13,806 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:13,807 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:13,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,816 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T05:45:13,816 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:13,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:13,818 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,825 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,826 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,833 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,835 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,840 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T05:45:13,843 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:13,857 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T05:45:13,858 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68327872, jitterRate=0.018164634704589844}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T05:45:13,868 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732686313702Initializing all the Stores at 1732686313703 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686313704 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686313705 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686313706 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686313706Cleaning up temporary data from old regions at 1732686313835 (+129 ms)Region opened successfully at 1732686313867 (+32 ms) 2024-11-27T05:45:13,871 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T05:45:13,930 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c34e25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4da2d47fc9ac/172.17.0.2:0 2024-11-27T05:45:13,989 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T05:45:14,010 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T05:45:14,011 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T05:45:14,017 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T05:45:14,018 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T05:45:14,028 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-27T05:45:14,028 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T05:45:14,068 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T05:45:14,081 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T05:45:14,129 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T05:45:14,135 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T05:45:14,138 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T05:45:14,150 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T05:45:14,154 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T05:45:14,161 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T05:45:14,171 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T05:45:14,174 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T05:45:14,181 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T05:45:14,213 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T05:45:14,224 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T05:45:14,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:14,252 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:14,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:14,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,258 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4da2d47fc9ac,40489,1732686311548, sessionid=0x1017ad2eae10000, setting cluster-up flag (Was=false) 2024-11-27T05:45:14,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,329 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T05:45:14,332 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:14,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,358 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:14,392 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T05:45:14,394 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:14,399 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T05:45:14,463 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(746): ClusterId : 678c977a-db27-498d-a76e-34e244be1efe 2024-11-27T05:45:14,463 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(746): ClusterId : 678c977a-db27-498d-a76e-34e244be1efe 2024-11-27T05:45:14,466 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T05:45:14,466 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T05:45:14,488 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T05:45:14,489 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T05:45:14,498 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T05:45:14,498 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T05:45:14,508 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T05:45:14,509 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T05:45:14,509 DEBUG [RS:0;4da2d47fc9ac:44713 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4195827d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4da2d47fc9ac/172.17.0.2:0 2024-11-27T05:45:14,510 DEBUG [RS:1;4da2d47fc9ac:37167 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126e22d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4da2d47fc9ac/172.17.0.2:0 2024-11-27T05:45:14,511 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:14,522 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T05:45:14,526 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4da2d47fc9ac:37167 2024-11-27T05:45:14,529 DEBUG [RS:0;4da2d47fc9ac:44713 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4da2d47fc9ac:44713 2024-11-27T05:45:14,530 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T05:45:14,530 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T05:45:14,530 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T05:45:14,530 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T05:45:14,530 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T05:45:14,530 DEBUG [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T05:45:14,533 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T05:45:14,533 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(2659): reportForDuty to master=4da2d47fc9ac,40489,1732686311548 with port=37167, startcode=1732686312774 2024-11-27T05:45:14,534 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(2659): reportForDuty to master=4da2d47fc9ac,40489,1732686311548 with port=44713, startcode=1732686312525 2024-11-27T05:45:14,541 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4da2d47fc9ac,40489,1732686311548 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T05:45:14,551 DEBUG [RS:0;4da2d47fc9ac:44713 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T05:45:14,551 DEBUG [RS:1;4da2d47fc9ac:37167 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T05:45:14,557 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:14,557 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:14,557 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:14,558 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:14,558 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4da2d47fc9ac:0, corePoolSize=10, maxPoolSize=10 2024-11-27T05:45:14,558 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,558 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=2, maxPoolSize=2 2024-11-27T05:45:14,559 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,567 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:14,568 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T05:45:14,570 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732686344569 2024-11-27T05:45:14,572 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T05:45:14,573 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:14,573 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T05:45:14,574 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T05:45:14,576 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T05:45:14,577 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T05:45:14,577 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T05:45:14,577 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T05:45:14,578 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,594 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T05:45:14,595 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T05:45:14,596 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T05:45:14,600 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T05:45:14,601 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T05:45:14,624 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.large.0-1732686314602,5,FailOnTimeoutGroup] 2024-11-27T05:45:14,648 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.small.0-1732686314625,5,FailOnTimeoutGroup] 2024-11-27T05:45:14,648 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,649 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T05:45:14,650 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,651 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,672 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57453, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T05:45:14,673 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35613, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T05:45:14,681 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40489 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:14,687 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T05:45:14,688 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:14,684 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40489 {}] master.ServerManager(517): Registering regionserver=4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:14,707 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40489 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:14,708 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40489 {}] master.ServerManager(517): Registering regionserver=4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:14,712 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:14,713 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T05:45:14,713 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T05:45:14,717 DEBUG [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:14,717 DEBUG [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T05:45:14,717 DEBUG [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T05:45:14,722 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:14,724 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T05:45:14,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T05:45:14,750 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T05:45:14,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:14,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:14,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T05:45:14,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T05:45:14,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:14,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:14,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T05:45:14,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T05:45:14,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:14,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:14,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T05:45:14,784 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T05:45:14,784 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:14,785 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:14,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T05:45:14,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:14,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:14,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T05:45:14,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T05:45:14,788 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T05:45:14,790 DEBUG [RS:1;4da2d47fc9ac:37167 {}] zookeeper.ZKUtil(111): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:14,791 WARN [RS:1;4da2d47fc9ac:37167 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T05:45:14,791 INFO [RS:1;4da2d47fc9ac:37167 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:14,791 DEBUG [RS:0;4da2d47fc9ac:44713 {}] zookeeper.ZKUtil(111): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:14,791 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:14,792 WARN [RS:0;4da2d47fc9ac:44713 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T05:45:14,792 INFO [RS:0;4da2d47fc9ac:44713 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:14,792 DEBUG [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:14,795 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4da2d47fc9ac,44713,1732686312525] 2024-11-27T05:45:14,795 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4da2d47fc9ac,37167,1732686312774] 2024-11-27T05:45:14,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T05:45:14,808 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T05:45:14,809 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68257637, jitterRate=0.017118051648139954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T05:45:14,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732686314723Initializing all the Stores at 1732686314723Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686314723Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686314724 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686314724Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686314724Cleaning up temporary data from old regions at 1732686314788 (+64 ms)Region opened successfully at 1732686314815 (+27 ms) 2024-11-27T05:45:14,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T05:45:14,815 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T05:45:14,815 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T05:45:14,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T05:45:14,816 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T05:45:14,817 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:14,817 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732686314815Disabling compacts and flushes for region at 1732686314815Disabling writes for close at 1732686314816 (+1 ms)Writing region close event to WAL at 1732686314817 (+1 ms)Closed at 1732686314817 2024-11-27T05:45:14,820 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:14,820 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T05:45:14,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T05:45:14,839 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T05:45:14,840 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T05:45:14,840 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T05:45:14,842 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T05:45:14,867 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T05:45:14,868 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T05:45:14,880 INFO [RS:0;4da2d47fc9ac:44713 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T05:45:14,880 INFO [RS:1;4da2d47fc9ac:37167 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T05:45:14,880 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,880 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,884 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T05:45:14,884 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T05:45:14,891 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T05:45:14,893 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,894 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,894 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,894 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,895 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T05:45:14,895 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,895 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,895 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,895 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=2, maxPoolSize=2 2024-11-27T05:45:14,895 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,896 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,896 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,896 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,896 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,896 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=2, maxPoolSize=2 2024-11-27T05:45:14,897 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,897 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,897 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,897 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,897 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,898 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,898 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,898 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:14,898 DEBUG [RS:1;4da2d47fc9ac:37167 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:14,900 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,900 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,900 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,900 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,901 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:14,901 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:14,901 DEBUG [RS:0;4da2d47fc9ac:44713 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:14,904 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,905 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,905 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,905 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,905 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,905 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,37167,1732686312774-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T05:45:14,910 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,911 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,911 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,911 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,911 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,911 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,44713,1732686312525-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T05:45:14,937 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T05:45:14,938 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T05:45:14,940 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,44713,1732686312525-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,940 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,37167,1732686312774-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,940 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,940 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.Replication(171): 4da2d47fc9ac,37167,1732686312774 started 2024-11-27T05:45:14,942 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,943 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.Replication(171): 4da2d47fc9ac,44713,1732686312525 started 2024-11-27T05:45:14,968 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,968 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:14,969 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1482): Serving as 4da2d47fc9ac,37167,1732686312774, RpcServer on 4da2d47fc9ac/172.17.0.2:37167, sessionid=0x1017ad2eae10002 2024-11-27T05:45:14,969 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(1482): Serving as 4da2d47fc9ac,44713,1732686312525, RpcServer on 4da2d47fc9ac/172.17.0.2:44713, sessionid=0x1017ad2eae10001 2024-11-27T05:45:14,970 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T05:45:14,970 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T05:45:14,970 DEBUG [RS:0;4da2d47fc9ac:44713 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:14,970 DEBUG [RS:1;4da2d47fc9ac:37167 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:14,971 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,44713,1732686312525' 2024-11-27T05:45:14,971 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,37167,1732686312774' 2024-11-27T05:45:14,971 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T05:45:14,971 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T05:45:14,972 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T05:45:14,972 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T05:45:14,973 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T05:45:14,973 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T05:45:14,973 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T05:45:14,973 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T05:45:14,974 DEBUG [RS:0;4da2d47fc9ac:44713 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:14,974 DEBUG [RS:1;4da2d47fc9ac:37167 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:14,974 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,44713,1732686312525' 2024-11-27T05:45:14,974 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T05:45:14,974 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,37167,1732686312774' 2024-11-27T05:45:14,974 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T05:45:14,975 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T05:45:14,975 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T05:45:14,976 DEBUG [RS:0;4da2d47fc9ac:44713 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T05:45:14,976 DEBUG [RS:1;4da2d47fc9ac:37167 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T05:45:14,976 INFO [RS:1;4da2d47fc9ac:37167 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T05:45:14,976 INFO [RS:0;4da2d47fc9ac:44713 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T05:45:14,976 INFO [RS:0;4da2d47fc9ac:44713 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T05:45:14,976 INFO [RS:1;4da2d47fc9ac:37167 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T05:45:14,993 WARN [4da2d47fc9ac:40489 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T05:45:15,081 INFO [RS:1;4da2d47fc9ac:37167 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T05:45:15,081 INFO [RS:0;4da2d47fc9ac:44713 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T05:45:15,083 INFO [RS:0;4da2d47fc9ac:44713 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C44713%2C1732686312525, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,44713,1732686312525, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:15,083 INFO [RS:1;4da2d47fc9ac:37167 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C37167%2C1732686312774, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37167,1732686312774, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:15,107 INFO [RS:0;4da2d47fc9ac:44713 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,44713,1732686312525/4da2d47fc9ac%2C44713%2C1732686312525.1732686315086 2024-11-27T05:45:15,107 INFO [RS:1;4da2d47fc9ac:37167 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37167,1732686312774/4da2d47fc9ac%2C37167%2C1732686312774.1732686315086 2024-11-27T05:45:15,108 DEBUG [RS:0;4da2d47fc9ac:44713 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:15,108 DEBUG [RS:1;4da2d47fc9ac:37167 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:15,246 DEBUG [4da2d47fc9ac:40489 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-27T05:45:15,259 DEBUG [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(204): Hosts are {4da2d47fc9ac=0} racks are {/default-rack=0} 2024-11-27T05:45:15,262 DEBUG [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T05:45:15,262 DEBUG [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T05:45:15,262 DEBUG [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T05:45:15,262 DEBUG [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T05:45:15,262 INFO [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T05:45:15,262 INFO [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T05:45:15,262 DEBUG [4da2d47fc9ac:40489 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T05:45:15,269 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:15,275 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4da2d47fc9ac,37167,1732686312774, state=OPENING 2024-11-27T05:45:15,287 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T05:45:15,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:15,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:15,297 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:15,298 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:15,298 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:15,298 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:15,300 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T05:45:15,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,37167,1732686312774}] 2024-11-27T05:45:15,476 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T05:45:15,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42337, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T05:45:15,494 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T05:45:15,495 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:15,496 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T05:45:15,499 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C37167%2C1732686312774.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37167,1732686312774, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:15,540 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37167,1732686312774/4da2d47fc9ac%2C37167%2C1732686312774.meta.1732686315501.meta 2024-11-27T05:45:15,541 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:15,542 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:15,545 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T05:45:15,549 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T05:45:15,556 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T05:45:15,562 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T05:45:15,563 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:15,563 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T05:45:15,564 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T05:45:15,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T05:45:15,567 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T05:45:15,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:15,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:15,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T05:45:15,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T05:45:15,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:15,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:15,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T05:45:15,571 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T05:45:15,572 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:15,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:15,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T05:45:15,573 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T05:45:15,573 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:15,574 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:15,574 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T05:45:15,576 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:15,577 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:15,578 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T05:45:15,578 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T05:45:15,578 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T05:45:15,580 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T05:45:15,581 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69258814, jitterRate=0.03203675150871277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T05:45:15,582 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T05:45:15,586 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732686315565Writing region info on filesystem at 1732686315565Initializing all the Stores at 1732686315565Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686315565Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686315566 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686315566Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686315566Cleaning up temporary data from old regions at 1732686315578 (+12 ms)Running coprocessor post-open hooks at 1732686315582 (+4 ms)Region opened successfully at 1732686315585 (+3 ms) 2024-11-27T05:45:15,595 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732686315467 2024-11-27T05:45:15,611 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T05:45:15,611 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T05:45:15,613 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:15,615 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4da2d47fc9ac,37167,1732686312774, state=OPEN 2024-11-27T05:45:15,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:15,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:15,645 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:15,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:15,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:15,645 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:15,646 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:15,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T05:45:15,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,37167,1732686312774 in 344 msec 2024-11-27T05:45:15,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T05:45:15,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 824 msec 2024-11-27T05:45:15,658 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:15,658 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T05:45:15,695 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T05:45:15,697 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,37167,1732686312774, seqNum=-1] 2024-11-27T05:45:15,723 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T05:45:15,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50803, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T05:45:15,758 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3160 sec 2024-11-27T05:45:15,758 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732686315758, completionTime=-1 2024-11-27T05:45:15,762 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-27T05:45:15,762 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T05:45:15,803 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T05:45:15,803 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732686375803 2024-11-27T05:45:15,803 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732686435803 2024-11-27T05:45:15,803 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 40 msec 2024-11-27T05:45:15,806 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-27T05:45:15,830 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,40489,1732686311548-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:15,830 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,40489,1732686311548-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:15,830 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,40489,1732686311548-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:15,832 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4da2d47fc9ac:40489, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:15,833 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:15,834 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:15,840 DEBUG [master/4da2d47fc9ac:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T05:45:15,868 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.801sec 2024-11-27T05:45:15,870 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T05:45:15,872 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T05:45:15,873 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T05:45:15,873 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T05:45:15,873 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T05:45:15,874 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,40489,1732686311548-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T05:45:15,875 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,40489,1732686311548-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T05:45:15,876 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T05:45:15,878 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T05:45:15,878 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,40489,1732686311548-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:15,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109a9e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T05:45:15,976 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T05:45:15,976 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T05:45:15,980 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4da2d47fc9ac,40489,-1 for getting cluster id 2024-11-27T05:45:15,983 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T05:45:15,995 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '678c977a-db27-498d-a76e-34e244be1efe' 2024-11-27T05:45:15,998 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T05:45:15,999 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "678c977a-db27-498d-a76e-34e244be1efe" 2024-11-27T05:45:15,999 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@213c6fe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T05:45:16,000 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4da2d47fc9ac,40489,-1] 2024-11-27T05:45:16,003 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T05:45:16,005 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:16,007 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59312, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T05:45:16,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36c35605, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T05:45:16,011 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T05:45:16,019 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,37167,1732686312774, seqNum=-1] 2024-11-27T05:45:16,020 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T05:45:16,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T05:45:16,056 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:16,064 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T05:45:16,070 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:16,073 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6034ffa3 2024-11-27T05:45:16,075 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T05:45:16,078 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T05:45:16,085 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732686316056', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T05:45:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 2024-11-27T05:45:16,098 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T05:45:16,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732686316056" procId is: 4 2024-11-27T05:45:16,101 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:16,103 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T05:45:16,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:16,137 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d2d80eaec449931e4f467b723a23c6bd, NAME => 'testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732686316056', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:16,145 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:16,145 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(1722): Closing d2d80eaec449931e4f467b723a23c6bd, disabling compactions & flushes 2024-11-27T05:45:16,145 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:16,145 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:16,145 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. after waiting 0 ms 2024-11-27T05:45:16,146 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:16,146 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:16,146 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732686316056-pool-0 {}] regionserver.HRegion(1676): Region close journal for d2d80eaec449931e4f467b723a23c6bd: Waiting for close lock at 1732686316145Disabling compacts and flushes for region at 1732686316145Disabling writes for close at 1732686316146 (+1 ms)Writing region close event to WAL at 1732686316146Closed at 1732686316146 2024-11-27T05:45:16,148 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T05:45:16,153 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732686316148"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732686316148"}]},"ts":"1732686316148"} 2024-11-27T05:45:16,157 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T05:45:16,159 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T05:45:16,162 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732686316056","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732686316159"}]},"ts":"1732686316159"} 2024-11-27T05:45:16,166 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732686316056, state=ENABLING in hbase:meta 2024-11-27T05:45:16,167 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4da2d47fc9ac=0} racks are {/default-rack=0} 2024-11-27T05:45:16,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T05:45:16,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T05:45:16,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T05:45:16,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T05:45:16,168 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T05:45:16,168 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T05:45:16,168 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T05:45:16,170 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, ASSIGN}] 2024-11-27T05:45:16,172 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, ASSIGN 2024-11-27T05:45:16,173 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, ASSIGN; state=OFFLINE, location=4da2d47fc9ac,44713,1732686312525; forceNewPlan=false, retain=false 2024-11-27T05:45:16,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:16,326 INFO [4da2d47fc9ac:40489 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T05:45:16,327 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2d80eaec449931e4f467b723a23c6bd, regionState=OPENING, regionLocation=4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:16,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, ASSIGN because future has completed 2024-11-27T05:45:16,333 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,44713,1732686312525}] 2024-11-27T05:45:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:16,486 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T05:45:16,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54691, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T05:45:16,505 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:16,506 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d2d80eaec449931e4f467b723a23c6bd, NAME => 'testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:16,506 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732686316056 d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,507 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:16,507 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,507 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,510 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,522 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2d80eaec449931e4f467b723a23c6bd columnFamilyName family 2024-11-27T05:45:16,523 DEBUG [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:16,523 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.HStore(327): Store=d2d80eaec449931e4f467b723a23c6bd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:16,524 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,525 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,525 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,526 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,526 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,527 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,539 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T05:45:16,540 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d2d80eaec449931e4f467b723a23c6bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62195760, jitterRate=-0.0732109546661377}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T05:45:16,540 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:16,541 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d2d80eaec449931e4f467b723a23c6bd: Running coprocessor pre-open hook at 1732686316507Writing region info on filesystem at 1732686316507Initializing all the Stores at 1732686316508 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686316508Cleaning up temporary data from old regions at 1732686316526 (+18 ms)Running coprocessor post-open hooks at 1732686316540 (+14 ms)Region opened successfully at 1732686316541 (+1 ms) 2024-11-27T05:45:16,543 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd., pid=6, masterSystemTime=1732686316485 2024-11-27T05:45:16,547 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:16,547 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:16,547 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d2d80eaec449931e4f467b723a23c6bd, regionState=OPEN, openSeqNum=2, regionLocation=4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:16,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,44713,1732686312525 because future has completed 2024-11-27T05:45:16,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T05:45:16,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,44713,1732686312525 in 226 msec 2024-11-27T05:45:16,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T05:45:16,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, ASSIGN in 391 msec 2024-11-27T05:45:16,566 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T05:45:16,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732686316056","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732686316566"}]},"ts":"1732686316566"} 2024-11-27T05:45:16,570 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732686316056, state=ENABLED in hbase:meta 2024-11-27T05:45:16,572 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T05:45:16,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732686316056 in 483 msec 2024-11-27T05:45:16,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:16,736 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732686316056 get assigned. Timeout = 60000ms 2024-11-27T05:45:16,736 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732686316056 completed 2024-11-27T05:45:16,740 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T05:45:16,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732686316056 assigned to meta. Checking AM states. 2024-11-27T05:45:16,747 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T05:45:16,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732686316056 assigned. 2024-11-27T05:45:16,759 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732686316056', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd., hostname=4da2d47fc9ac,44713,1732686312525, seqNum=2] 2024-11-27T05:45:16,761 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T05:45:16,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T05:45:16,775 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d2d80eaec449931e4f467b723a23c6bd 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-27T05:45:16,811 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 821bb22559d34b2bba00f996163ddf41 2024-11-27T05:45:16,860 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/.tmp/family/821bb22559d34b2bba00f996163ddf41 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-27T05:45:16,866 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/.tmp/family/821bb22559d34b2bba00f996163ddf41 2024-11-27T05:45:16,909 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/.tmp/family/821bb22559d34b2bba00f996163ddf41 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41 2024-11-27T05:45:16,911 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41, delay=921 ms 2024-11-27T05:45:16,914 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41, entries=2, sequenceid=6, filesize=4.9 K 2024-11-27T05:45:16,922 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d2d80eaec449931e4f467b723a23c6bd in 145ms, sequenceid=6, compaction requested=false 2024-11-27T05:45:16,923 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732686316056' 2024-11-27T05:45:16,926 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d2d80eaec449931e4f467b723a23c6bd: 2024-11-27T05:45:17,835 DEBUG [hfile-prefetch-1732686316911 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 821bb22559d34b2bba00f996163ddf41_0. 2024-11-27T05:45:17,841 DEBUG [hfile-prefetch-1732686316911 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 821bb22559d34b2bba00f996163ddf41_117. 2024-11-27T05:45:17,842 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41, with total blocks 2, and data blocks 1 2024-11-27T05:45:17,842 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-27T05:45:17,843 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-27T05:45:17,843 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41. Will sleep 100ms and try the verification again. 2024-11-27T05:45:17,884 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-27T05:45:17,885 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-27T05:45:17,885 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:17,885 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T05:45:17,904 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T05:45:17,905 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732686317885 2024-11-27T05:45:17,905 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-27T05:45:17,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=d2d80eaec449931e4f467b723a23c6bd, source=4da2d47fc9ac,44713,1732686312525, destination=4da2d47fc9ac,37167,1732686312774, warming up region on 4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:17,943 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41, with total blocks 2, and data blocks 1 2024-11-27T05:45:17,944 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-27T05:45:17,944 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=d2d80eaec449931e4f467b723a23c6bd, source=4da2d47fc9ac,44713,1732686312525, destination=4da2d47fc9ac,37167,1732686312774, running balancer 2024-11-27T05:45:17,944 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2293): found block for file 821bb22559d34b2bba00f996163ddf41 in the backing map. Acquiring read lock for offset 0 2024-11-27T05:45:17,944 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41 has now been fully cached. 2024-11-27T05:45:17,944 DEBUG [hfile-prefetch-1732686316911 {}] hfile.PrefetchExecutor(121): Prefetch completed for 821bb22559d34b2bba00f996163ddf41 2024-11-27T05:45:17,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, REOPEN/MOVE 2024-11-27T05:45:17,945 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, REOPEN/MOVE 2024-11-27T05:45:17,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(7855): Warmup {ENCODED => d2d80eaec449931e4f467b723a23c6bd, NAME => 'testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:17,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:17,947 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:17,948 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2d80eaec449931e4f467b723a23c6bd, regionState=CLOSING, regionLocation=4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:17,948 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2d80eaec449931e4f467b723a23c6bd columnFamilyName family 2024-11-27T05:45:17,948 DEBUG [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:17,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, REOPEN/MOVE because future has completed 2024-11-27T05:45:17,953 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-27T05:45:17,953 DEBUG [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41 2024-11-27T05:45:17,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,44713,1732686312525}] 2024-11-27T05:45:17,953 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.HStore(327): Store=d2d80eaec449931e4f467b723a23c6bd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:17,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(1722): Closing d2d80eaec449931e4f467b723a23c6bd, disabling compactions & flushes 2024-11-27T05:45:17,953 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. after waiting 0 ms 2024-11-27T05:45:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:17,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:17,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37167 {}] regionserver.HRegion(1676): Region close journal for d2d80eaec449931e4f467b723a23c6bd: Waiting for close lock at 1732686317953Disabling compacts and flushes for region at 1732686317953Disabling writes for close at 1732686317954 (+1 ms)Writing region close event to WAL at 1732686317959 (+5 ms)Closed at 1732686317959 2024-11-27T05:45:17,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40489 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-27T05:45:18,114 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,115 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-27T05:45:18,115 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing d2d80eaec449931e4f467b723a23c6bd, disabling compactions & flushes 2024-11-27T05:45:18,116 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,116 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,116 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. after waiting 0 ms 2024-11-27T05:45:18,116 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,118 DEBUG [StoreFileCloser-d2d80eaec449931e4f467b723a23c6bd-family-1 {}] bucket.BucketCache(768): Removing file 821bb22559d34b2bba00f996163ddf41 for region d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,118 DEBUG [StoreFileCloser-d2d80eaec449931e4f467b723a23c6bd-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 821bb22559d34b2bba00f996163ddf41, starting offset: 0, end offset: 9223372036854775807 2024-11-27T05:45:18,120 DEBUG [StoreFileCloser-d2d80eaec449931e4f467b723a23c6bd-family-1 {}] bucket.BucketCache(856): removed key 821bb22559d34b2bba00f996163ddf41_0 from back map with offset lock 2094080 in the evict process 2024-11-27T05:45:18,125 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-27T05:45:18,126 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,126 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for d2d80eaec449931e4f467b723a23c6bd: Waiting for close lock at 1732686318115Running coprocessor pre-close hooks at 1732686318115Disabling compacts and flushes for region at 1732686318115Disabling writes for close at 1732686318116 (+1 ms)Writing region close event to WAL at 1732686318120 (+4 ms)Running coprocessor post-close hooks at 1732686318125 (+5 ms)Closed at 1732686318126 (+1 ms) 2024-11-27T05:45:18,127 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding d2d80eaec449931e4f467b723a23c6bd move to 4da2d47fc9ac,37167,1732686312774 record at close sequenceid=6 2024-11-27T05:45:18,130 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,131 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2d80eaec449931e4f467b723a23c6bd, regionState=CLOSED 2024-11-27T05:45:18,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,44713,1732686312525 because future has completed 2024-11-27T05:45:18,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T05:45:18,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,44713,1732686312525 in 185 msec 2024-11-27T05:45:18,142 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, REOPEN/MOVE; state=CLOSED, location=4da2d47fc9ac,37167,1732686312774; forceNewPlan=false, retain=false 2024-11-27T05:45:18,172 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-27T05:45:18,172 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-27T05:45:18,173 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:18,173 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T05:45:18,177 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T05:45:18,177 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732686318172 2024-11-27T05:45:18,177 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-27T05:45:18,292 INFO [4da2d47fc9ac:40489 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T05:45:18,293 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2d80eaec449931e4f467b723a23c6bd, regionState=OPENING, regionLocation=4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:18,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, REOPEN/MOVE because future has completed 2024-11-27T05:45:18,297 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,37167,1732686312774}] 2024-11-27T05:45:18,454 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,454 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => d2d80eaec449931e4f467b723a23c6bd, NAME => 'testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:18,455 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732686316056 d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,455 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:18,455 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,455 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,456 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,458 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2d80eaec449931e4f467b723a23c6bd columnFamilyName family 2024-11-27T05:45:18,458 DEBUG [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:18,460 DEBUG [StoreFileOpener-d2d80eaec449931e4f467b723a23c6bd-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41, delay=996 ms 2024-11-27T05:45:18,462 DEBUG [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41 2024-11-27T05:45:18,463 INFO [StoreOpener-d2d80eaec449931e4f467b723a23c6bd-1 {}] regionserver.HStore(327): Store=d2d80eaec449931e4f467b723a23c6bd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:18,463 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,464 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,465 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,465 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,465 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,470 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,471 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened d2d80eaec449931e4f467b723a23c6bd; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65463711, jitterRate=-0.024514690041542053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T05:45:18,472 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,473 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for d2d80eaec449931e4f467b723a23c6bd: Running coprocessor pre-open hook at 1732686318455Writing region info on filesystem at 1732686318455Initializing all the Stores at 1732686318456 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686318456Cleaning up temporary data from old regions at 1732686318465 (+9 ms)Running coprocessor post-open hooks at 1732686318472 (+7 ms)Region opened successfully at 1732686318473 (+1 ms) 2024-11-27T05:45:18,474 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd., pid=9, masterSystemTime=1732686318450 2024-11-27T05:45:18,477 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,477 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=d2d80eaec449931e4f467b723a23c6bd, regionState=OPEN, openSeqNum=10, regionLocation=4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:18,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,37167,1732686312774 because future has completed 2024-11-27T05:45:18,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-27T05:45:18,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure d2d80eaec449931e4f467b723a23c6bd, server=4da2d47fc9ac,37167,1732686312774 in 186 msec 2024-11-27T05:45:18,488 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732686316056, region=d2d80eaec449931e4f467b723a23c6bd, REOPEN/MOVE in 542 msec 2024-11-27T05:45:18,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T05:45:18,499 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T05:45:18,499 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:18,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:18,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:18,505 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T05:45:18,505 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T05:45:18,505 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-27T05:45:18,505 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4da2d47fc9ac,40489,1732686311548 2024-11-27T05:45:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:18,557 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T05:45:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:18,558 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T05:45:18,558 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:18,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:18,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:18,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:18,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:18,559 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4da2d47fc9ac,44713,1732686312525' ***** 2024-11-27T05:45:18,559 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T05:45:18,559 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4da2d47fc9ac,37167,1732686312774' ***** 2024-11-27T05:45:18,559 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T05:45:18,559 INFO [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T05:45:18,559 INFO [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T05:45:18,559 INFO [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.persistence 2024-11-27T05:45:18,559 INFO [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/bucket.persistence 2024-11-27T05:45:18,559 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T05:45:18,559 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T05:45:18,560 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T05:45:18,560 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T05:45:18,560 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T05:45:18,560 INFO [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T05:45:18,560 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T05:45:18,564 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T05:45:18,564 INFO [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T05:45:18,565 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T05:45:18,570 DEBUG [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1487): Thread RS:1;4da2d47fc9ac:37167 started persisting bucket cache to file 2024-11-27T05:45:18,570 DEBUG [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:18,571 DEBUG [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T05:45:18,574 DEBUG [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T05:45:18,574 DEBUG [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732686318570 2024-11-27T05:45:18,574 DEBUG [RS:1;4da2d47fc9ac:37167 {}] bucket.BucketCache(1509): Thread RS:1;4da2d47fc9ac:37167 finished persisting bucket cache to file, renaming 2024-11-27T05:45:18,575 DEBUG [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1487): Thread RS:0;4da2d47fc9ac:44713 started persisting bucket cache to file 2024-11-27T05:45:18,609 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T05:45:18,611 INFO [regionserver/4da2d47fc9ac:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:18,614 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T05:45:18,623 INFO [regionserver/4da2d47fc9ac:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:18,637 DEBUG [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:18,637 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T05:45:18,638 DEBUG [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T05:45:18,638 INFO [RS:1;4da2d47fc9ac:37167 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T05:45:18,638 INFO [RS:1;4da2d47fc9ac:37167 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T05:45:18,638 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(3091): Received CLOSE for d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,639 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(959): stopping server 4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:18,639 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T05:45:18,639 INFO [RS:1;4da2d47fc9ac:37167 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4da2d47fc9ac:37167. 2024-11-27T05:45:18,639 DEBUG [RS:1;4da2d47fc9ac:37167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:18,639 DEBUG [RS:1;4da2d47fc9ac:37167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:18,639 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T05:45:18,639 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T05:45:18,639 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T05:45:18,640 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T05:45:18,640 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T05:45:18,640 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2d80eaec449931e4f467b723a23c6bd, disabling compactions & flushes 2024-11-27T05:45:18,640 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,640 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T05:45:18,640 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,640 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T05:45:18,640 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. after waiting 0 ms 2024-11-27T05:45:18,640 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, d2d80eaec449931e4f467b723a23c6bd=testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd.} 2024-11-27T05:45:18,640 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,640 DEBUG [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d2d80eaec449931e4f467b723a23c6bd 2024-11-27T05:45:18,641 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T05:45:18,641 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T05:45:18,641 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T05:45:18,641 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T05:45:18,641 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T05:45:18,641 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-27T05:45:18,642 DEBUG [StoreFileCloser-d2d80eaec449931e4f467b723a23c6bd-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/family/821bb22559d34b2bba00f996163ddf41 2024-11-27T05:45:18,645 DEBUG [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T05:45:18,645 DEBUG [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732686318575 2024-11-27T05:45:18,645 DEBUG [RS:0;4da2d47fc9ac:44713 {}] bucket.BucketCache(1509): Thread RS:0;4da2d47fc9ac:44713 finished persisting bucket cache to file, renaming 2024-11-27T05:45:18,646 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T05:45:18,646 INFO [RS:0;4da2d47fc9ac:44713 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T05:45:18,646 INFO [RS:0;4da2d47fc9ac:44713 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T05:45:18,646 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(959): stopping server 4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:18,646 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T05:45:18,646 INFO [RS:0;4da2d47fc9ac:44713 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4da2d47fc9ac:44713. 2024-11-27T05:45:18,646 DEBUG [RS:0;4da2d47fc9ac:44713 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:18,646 DEBUG [RS:0;4da2d47fc9ac:44713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:18,647 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(976): stopping server 4da2d47fc9ac,44713,1732686312525; all regions closed. 2024-11-27T05:45:18,648 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnRegionMove1732686316056/d2d80eaec449931e4f467b723a23c6bd/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-27T05:45:18,649 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,649 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2d80eaec449931e4f467b723a23c6bd: Waiting for close lock at 1732686318639Running coprocessor pre-close hooks at 1732686318640 (+1 ms)Disabling compacts and flushes for region at 1732686318640Disabling writes for close at 1732686318640Writing region close event to WAL at 1732686318643 (+3 ms)Running coprocessor post-close hooks at 1732686318648 (+5 ms)Closed at 1732686318648 2024-11-27T05:45:18,649 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd. 2024-11-27T05:45:18,653 DEBUG [RS:0;4da2d47fc9ac:44713 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:18,653 INFO [RS:0;4da2d47fc9ac:44713 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C44713%2C1732686312525:(num 1732686315086) 2024-11-27T05:45:18,653 DEBUG [RS:0;4da2d47fc9ac:44713 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:18,653 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:18,653 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T05:45:18,653 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.ChoreService(370): Chore service for: regionserver/4da2d47fc9ac:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T05:45:18,654 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T05:45:18,654 INFO [regionserver/4da2d47fc9ac:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T05:45:18,654 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T05:45:18,654 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T05:45:18,654 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T05:45:18,654 INFO [RS:0;4da2d47fc9ac:44713 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44713 2024-11-27T05:45:18,655 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 53f43cfc8a3442f092c40285568e7c7b 2024-11-27T05:45:18,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4da2d47fc9ac,44713,1732686312525 2024-11-27T05:45:18,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T05:45:18,673 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T05:45:18,674 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/info/53f43cfc8a3442f092c40285568e7c7b is 205, key is testBlockEvictionOnRegionMove1732686316056,,1732686316080.d2d80eaec449931e4f467b723a23c6bd./info:regioninfo/1732686318478/Put/seqid=0 2024-11-27T05:45:18,674 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4da2d47fc9ac,44713,1732686312525] 2024-11-27T05:45:18,675 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/info/53f43cfc8a3442f092c40285568e7c7b 2024-11-27T05:45:18,694 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e8414c4d1a9a4b5e8dd55beaf8b8f607 2024-11-27T05:45:18,697 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4da2d47fc9ac,44713,1732686312525 already deleted, retry=false 2024-11-27T05:45:18,697 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4da2d47fc9ac,44713,1732686312525 expired; onlineServers=1 2024-11-27T05:45:18,710 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/ns/e8414c4d1a9a4b5e8dd55beaf8b8f607 is 43, key is default/ns:d/1732686315731/Put/seqid=0 2024-11-27T05:45:18,711 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/ns/e8414c4d1a9a4b5e8dd55beaf8b8f607 2024-11-27T05:45:18,721 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07b440a470a948118e694de39dc7fc22 2024-11-27T05:45:18,736 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/table/07b440a470a948118e694de39dc7fc22 is 78, key is testBlockEvictionOnRegionMove1732686316056/table:state/1732686316566/Put/seqid=0 2024-11-27T05:45:18,737 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/table/07b440a470a948118e694de39dc7fc22 2024-11-27T05:45:18,741 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/info/53f43cfc8a3442f092c40285568e7c7b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/53f43cfc8a3442f092c40285568e7c7b 2024-11-27T05:45:18,742 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/53f43cfc8a3442f092c40285568e7c7b, delay=941 ms 2024-11-27T05:45:18,744 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/53f43cfc8a3442f092c40285568e7c7b, entries=15, sequenceid=15, filesize=7.6 K 2024-11-27T05:45:18,745 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/ns/e8414c4d1a9a4b5e8dd55beaf8b8f607 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/e8414c4d1a9a4b5e8dd55beaf8b8f607 2024-11-27T05:45:18,746 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/e8414c4d1a9a4b5e8dd55beaf8b8f607, delay=905 ms 2024-11-27T05:45:18,748 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/e8414c4d1a9a4b5e8dd55beaf8b8f607, entries=2, sequenceid=15, filesize=5.0 K 2024-11-27T05:45:18,749 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/table/07b440a470a948118e694de39dc7fc22 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/07b440a470a948118e694de39dc7fc22 2024-11-27T05:45:18,751 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/07b440a470a948118e694de39dc7fc22, delay=988 ms 2024-11-27T05:45:18,753 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/07b440a470a948118e694de39dc7fc22, entries=2, sequenceid=15, filesize=5.3 K 2024-11-27T05:45:18,754 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 113ms, sequenceid=15, compaction requested=false 2024-11-27T05:45:18,754 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T05:45:18,756 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/53f43cfc8a3442f092c40285568e7c7b 2024-11-27T05:45:18,757 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/e8414c4d1a9a4b5e8dd55beaf8b8f607 2024-11-27T05:45:18,757 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/07b440a470a948118e694de39dc7fc22 2024-11-27T05:45:18,763 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-27T05:45:18,763 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T05:45:18,764 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:18,764 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732686318640Running coprocessor pre-close hooks at 1732686318640Disabling compacts and flushes for region at 1732686318641 (+1 ms)Disabling writes for close at 1732686318641Obtaining lock to block concurrent updates at 1732686318642 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732686318642Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732686318642Flushing stores of hbase:meta,,1.1588230740 at 1732686318643 (+1 ms)Flushing 1588230740/info: creating writer at 1732686318643Flushing 1588230740/info: appending metadata at 1732686318673 (+30 ms)Flushing 1588230740/info: closing flushed file at 1732686318673Flushing 1588230740/ns: creating writer at 1732686318680 (+7 ms)Flushing 1588230740/ns: appending metadata at 1732686318709 (+29 ms)Flushing 1588230740/ns: closing flushed file at 1732686318709Flushing 1588230740/table: creating writer at 1732686318714 (+5 ms)Flushing 1588230740/table: appending metadata at 1732686318735 (+21 ms)Flushing 1588230740/table: closing flushed file at 1732686318736 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7fcd981f: reopening flushed file at 1732686318740 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27c871a5: reopening flushed file at 1732686318744 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f828b3e: reopening flushed file at 1732686318748 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 113ms, sequenceid=15, compaction requested=false at 1732686318754 (+6 ms)Writing region close event to WAL at 1732686318758 (+4 ms)Running coprocessor post-close hooks at 1732686318763 (+5 ms)Closed at 1732686318764 (+1 ms) 2024-11-27T05:45:18,765 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:18,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:18,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44713-0x1017ad2eae10001, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:18,787 INFO [RS:0;4da2d47fc9ac:44713 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T05:45:18,788 INFO [RS:0;4da2d47fc9ac:44713 {}] regionserver.HRegionServer(1031): Exiting; stopping=4da2d47fc9ac,44713,1732686312525; zookeeper connection closed. 2024-11-27T05:45:18,788 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5e84c36a {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5e84c36a 2024-11-27T05:45:18,841 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(976): stopping server 4da2d47fc9ac,37167,1732686312774; all regions closed. 2024-11-27T05:45:18,844 DEBUG [RS:1;4da2d47fc9ac:37167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:18,844 INFO [RS:1;4da2d47fc9ac:37167 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C37167%2C1732686312774.meta:.meta(num 1732686315501) 2024-11-27T05:45:18,846 DEBUG [RS:1;4da2d47fc9ac:37167 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:18,846 INFO [RS:1;4da2d47fc9ac:37167 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C37167%2C1732686312774:(num 1732686315086) 2024-11-27T05:45:18,846 DEBUG [RS:1;4da2d47fc9ac:37167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:18,846 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:18,846 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T05:45:18,846 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.ChoreService(370): Chore service for: regionserver/4da2d47fc9ac:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T05:45:18,847 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T05:45:18,847 INFO [regionserver/4da2d47fc9ac:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T05:45:18,847 INFO [RS:1;4da2d47fc9ac:37167 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37167 2024-11-27T05:45:18,855 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T05:45:18,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4da2d47fc9ac,37167,1732686312774 2024-11-27T05:45:18,855 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T05:45:18,866 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4da2d47fc9ac,37167,1732686312774] 2024-11-27T05:45:18,876 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4da2d47fc9ac,37167,1732686312774 already deleted, retry=false 2024-11-27T05:45:18,876 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4da2d47fc9ac,37167,1732686312774 expired; onlineServers=0 2024-11-27T05:45:18,876 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4da2d47fc9ac,40489,1732686311548' ***** 2024-11-27T05:45:18,876 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T05:45:18,877 INFO [M:0;4da2d47fc9ac:40489 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T05:45:18,877 INFO [M:0;4da2d47fc9ac:40489 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T05:45:18,877 DEBUG [M:0;4da2d47fc9ac:40489 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T05:45:18,877 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T05:45:18,877 DEBUG [M:0;4da2d47fc9ac:40489 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T05:45:18,877 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.small.0-1732686314625 {}] cleaner.HFileCleaner(306): Exit Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.small.0-1732686314625,5,FailOnTimeoutGroup] 2024-11-27T05:45:18,877 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.large.0-1732686314602 {}] cleaner.HFileCleaner(306): Exit Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.large.0-1732686314602,5,FailOnTimeoutGroup] 2024-11-27T05:45:18,877 INFO [M:0;4da2d47fc9ac:40489 {}] hbase.ChoreService(370): Chore service for: master/4da2d47fc9ac:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T05:45:18,877 INFO [M:0;4da2d47fc9ac:40489 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T05:45:18,878 DEBUG [M:0;4da2d47fc9ac:40489 {}] master.HMaster(1795): Stopping service threads 2024-11-27T05:45:18,878 INFO [M:0;4da2d47fc9ac:40489 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T05:45:18,878 INFO [M:0;4da2d47fc9ac:40489 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T05:45:18,878 INFO [M:0;4da2d47fc9ac:40489 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T05:45:18,879 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T05:45:18,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T05:45:18,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:18,887 DEBUG [M:0;4da2d47fc9ac:40489 {}] zookeeper.ZKUtil(347): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T05:45:18,887 WARN [M:0;4da2d47fc9ac:40489 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T05:45:18,888 INFO [M:0;4da2d47fc9ac:40489 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.lastflushedseqids 2024-11-27T05:45:18,901 INFO [M:0;4da2d47fc9ac:40489 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T05:45:18,901 INFO [M:0;4da2d47fc9ac:40489 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T05:45:18,902 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T05:45:18,902 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:18,902 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:18,902 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T05:45:18,902 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:18,902 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-27T05:45:18,913 DEBUG [M:0;4da2d47fc9ac:40489 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 860806320a47429e80e54a871e237d9f 2024-11-27T05:45:18,929 DEBUG [M:0;4da2d47fc9ac:40489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/860806320a47429e80e54a871e237d9f is 82, key is hbase:meta,,1/info:regioninfo/1732686315612/Put/seqid=0 2024-11-27T05:45:18,929 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/860806320a47429e80e54a871e237d9f 2024-11-27T05:45:18,939 DEBUG [M:0;4da2d47fc9ac:40489 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8b1915d748f247d69101dd9b1d91042a 2024-11-27T05:45:18,956 DEBUG [M:0;4da2d47fc9ac:40489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8b1915d748f247d69101dd9b1d91042a is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732686316574/Put/seqid=0 2024-11-27T05:45:18,957 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8b1915d748f247d69101dd9b1d91042a 2024-11-27T05:45:18,966 INFO [RS:1;4da2d47fc9ac:37167 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T05:45:18,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:18,966 INFO [RS:1;4da2d47fc9ac:37167 {}] regionserver.HRegionServer(1031): Exiting; stopping=4da2d47fc9ac,37167,1732686312774; zookeeper connection closed. 2024-11-27T05:45:18,966 DEBUG [M:0;4da2d47fc9ac:40489 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d569fc28cfe34a249860ef2eb8496496 2024-11-27T05:45:18,966 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37167-0x1017ad2eae10002, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:18,967 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5ff4fa2e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5ff4fa2e 2024-11-27T05:45:18,967 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-27T05:45:18,980 DEBUG [M:0;4da2d47fc9ac:40489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d569fc28cfe34a249860ef2eb8496496 is 69, key is 4da2d47fc9ac,37167,1732686312774/rs:state/1732686314692/Put/seqid=0 2024-11-27T05:45:18,981 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d569fc28cfe34a249860ef2eb8496496 2024-11-27T05:45:18,983 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/860806320a47429e80e54a871e237d9f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/860806320a47429e80e54a871e237d9f 2024-11-27T05:45:18,986 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/860806320a47429e80e54a871e237d9f, entries=8, sequenceid=84, filesize=5.5 K 2024-11-27T05:45:18,986 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8b1915d748f247d69101dd9b1d91042a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8b1915d748f247d69101dd9b1d91042a 2024-11-27T05:45:18,989 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8b1915d748f247d69101dd9b1d91042a, entries=9, sequenceid=84, filesize=6.5 K 2024-11-27T05:45:18,989 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d569fc28cfe34a249860ef2eb8496496 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d569fc28cfe34a249860ef2eb8496496 2024-11-27T05:45:18,992 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d569fc28cfe34a249860ef2eb8496496, entries=2, sequenceid=84, filesize=5.1 K 2024-11-27T05:45:18,993 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=84, compaction requested=false 2024-11-27T05:45:18,996 INFO [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:18,996 DEBUG [M:0;4da2d47fc9ac:40489 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732686318902Disabling compacts and flushes for region at 1732686318902Disabling writes for close at 1732686318902Obtaining lock to block concurrent updates at 1732686318902Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732686318902Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1732686318903 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732686318903Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732686318903Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732686318928 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732686318928Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732686318932 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732686318955 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732686318956 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732686318960 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732686318980 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732686318980Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1732686318983 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1732686318986 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1732686318989 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 91ms, sequenceid=84, compaction requested=false at 1732686318993 (+4 ms)Writing region close event to WAL at 1732686318996 (+3 ms)Closed at 1732686318996 2024-11-27T05:45:18,998 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T05:45:18,998 INFO [M:0;4da2d47fc9ac:40489 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T05:45:18,999 INFO [M:0;4da2d47fc9ac:40489 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40489 2024-11-27T05:45:18,999 INFO [M:0;4da2d47fc9ac:40489 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T05:45:19,110 INFO [M:0;4da2d47fc9ac:40489 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T05:45:19,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:19,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40489-0x1017ad2eae10000, quorum=127.0.0.1:54797, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:19,114 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T05:45:19,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-27T05:45:19,138 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,147 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/4da2d47fc9ac:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: hfile-prefetch-1732686316911 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/4da2d47fc9ac:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732686318746 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/4da2d47fc9ac:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732686318460 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/4da2d47fc9ac:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: hfile-prefetch-1732686318742 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=340 (was 300) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8064 (was 8499) 2024-11-27T05:45:19,154 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=340, ProcessCount=11, AvailableMemoryMB=8064 2024-11-27T05:45:19,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/hadoop-log-dir so I do NOT create it in target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195 2024-11-27T05:45:19,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/hadoop-tmp-dir so I do NOT create it in target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195 2024-11-27T05:45:19,155 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,158 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/cluster_9357c24a-46fe-a8da-1ff0-69c4c66819df, deleteOnExit=true 2024-11-27T05:45:19,161 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/cluster_9357c24a-46fe-a8da-1ff0-69c4c66819df/zookeeper_0, clientPort=56667, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/cluster_9357c24a-46fe-a8da-1ff0-69c4c66819df/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/cluster_9357c24a-46fe-a8da-1ff0-69c4c66819df/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T05:45:19,163 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56667 2024-11-27T05:45:19,175 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 with version=8 2024-11-27T05:45:19,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/hbase-staging 2024-11-27T05:45:19,178 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4da2d47fc9ac:0 server-side Connection retries=45 2024-11-27T05:45:19,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,179 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T05:45:19,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T05:45:19,180 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T05:45:19,180 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T05:45:19,181 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45173 2024-11-27T05:45:19,183 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45173 connecting to ZooKeeper ensemble=127.0.0.1:56667 2024-11-27T05:45:19,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:451730x0, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T05:45:19,259 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45173-0x1017ad309e40000 connected 2024-11-27T05:45:19,340 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,341 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,342 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:19,342 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93, hbase.cluster.distributed=false 2024-11-27T05:45:19,344 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T05:45:19,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45173 2024-11-27T05:45:19,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45173 2024-11-27T05:45:19,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45173 2024-11-27T05:45:19,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45173 2024-11-27T05:45:19,346 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45173 2024-11-27T05:45:19,364 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4da2d47fc9ac:0 server-side Connection retries=45 2024-11-27T05:45:19,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,364 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T05:45:19,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,364 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T05:45:19,365 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T05:45:19,365 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T05:45:19,366 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37641 2024-11-27T05:45:19,368 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37641 connecting to ZooKeeper ensemble=127.0.0.1:56667 2024-11-27T05:45:19,368 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,368 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,382 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:376410x0, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T05:45:19,383 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37641-0x1017ad309e40001 connected 2024-11-27T05:45:19,383 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:19,384 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T05:45:19,385 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.cache 2024-11-27T05:45:19,385 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T05:45:19,385 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T05:45:19,385 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T05:45:19,385 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T05:45:19,386 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T05:45:19,386 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T05:45:19,386 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T05:45:19,386 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T05:45:19,387 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence is complete. 2024-11-27T05:45:19,387 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T05:45:19,389 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T05:45:19,389 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37641 2024-11-27T05:45:19,389 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37641 2024-11-27T05:45:19,390 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37641 2024-11-27T05:45:19,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37641 2024-11-27T05:45:19,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37641 2024-11-27T05:45:19,411 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4da2d47fc9ac:0 server-side Connection retries=45 2024-11-27T05:45:19,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,411 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,411 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T05:45:19,412 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:19,412 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T05:45:19,412 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T05:45:19,412 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T05:45:19,432 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41581 2024-11-27T05:45:19,435 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41581 connecting to ZooKeeper ensemble=127.0.0.1:56667 2024-11-27T05:45:19,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415810x0, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T05:45:19,446 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:415810x0, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:19,446 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41581-0x1017ad309e40002 connected 2024-11-27T05:45:19,447 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T05:45:19,448 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.cache 2024-11-27T05:45:19,448 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T05:45:19,448 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T05:45:19,448 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T05:45:19,452 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T05:45:19,453 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-27T05:45:19,453 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T05:45:19,453 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence is complete. 2024-11-27T05:45:19,472 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T05:45:19,480 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T05:45:19,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T05:45:19,483 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T05:45:19,488 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41581 2024-11-27T05:45:19,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41581 2024-11-27T05:45:19,489 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41581 2024-11-27T05:45:19,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41581 2024-11-27T05:45:19,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41581 2024-11-27T05:45:19,507 DEBUG [M:0;4da2d47fc9ac:45173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4da2d47fc9ac:45173 2024-11-27T05:45:19,508 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:19,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:19,519 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:19,520 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:19,520 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:19,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,530 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T05:45:19,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T05:45:19,530 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,530 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,531 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T05:45:19,531 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4da2d47fc9ac,45173,1732686319178 from backup master directory 2024-11-27T05:45:19,540 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:19,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:19,540 WARN [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T05:45:19,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:19,540 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:19,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T05:45:19,542 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/hbase.id] with ID: 0968198c-6dbe-45d3-8c0d-7ebbe4604d27 2024-11-27T05:45:19,542 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.tmp/hbase.id 2024-11-27T05:45:19,545 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/hbase.id] 2024-11-27T05:45:19,579 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:19,580 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T05:45:19,582 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-27T05:45:19,593 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,598 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T05:45:19,600 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T05:45:19,600 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:19,620 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store 2024-11-27T05:45:19,628 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:19,628 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T05:45:19,628 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:19,628 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:19,628 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T05:45:19,628 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:19,628 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:19,629 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732686319628Disabling compacts and flushes for region at 1732686319628Disabling writes for close at 1732686319628Writing region close event to WAL at 1732686319628Closed at 1732686319628 2024-11-27T05:45:19,629 WARN [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/.initializing 2024-11-27T05:45:19,629 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/WALs/4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:19,635 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T05:45:19,639 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4da2d47fc9ac%2C45173%2C1732686319178, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/WALs/4da2d47fc9ac,45173,1732686319178, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/oldWALs, maxLogs=10 2024-11-27T05:45:19,658 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/WALs/4da2d47fc9ac,45173,1732686319178/4da2d47fc9ac%2C45173%2C1732686319178.1732686319639 2024-11-27T05:45:19,658 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:19,659 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:19,659 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:19,659 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,659 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,660 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T05:45:19,665 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,665 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:19,666 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T05:45:19,669 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,669 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:19,670 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,673 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T05:45:19,674 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:19,674 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T05:45:19,687 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,687 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:19,687 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,688 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,688 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,689 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,689 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,689 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T05:45:19,690 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T05:45:19,703 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T05:45:19,704 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72218811, jitterRate=0.07614414393901825}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T05:45:19,705 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732686319659Initializing all the Stores at 1732686319660 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686319660Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686319660Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686319660Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686319660Cleaning up temporary data from old regions at 1732686319689 (+29 ms)Region opened successfully at 1732686319705 (+16 ms) 2024-11-27T05:45:19,706 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T05:45:19,710 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1933532e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4da2d47fc9ac/172.17.0.2:0 2024-11-27T05:45:19,712 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T05:45:19,712 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T05:45:19,712 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T05:45:19,712 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T05:45:19,713 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-27T05:45:19,713 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-27T05:45:19,713 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T05:45:19,716 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T05:45:19,717 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T05:45:19,729 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T05:45:19,730 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T05:45:19,731 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T05:45:19,740 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T05:45:19,741 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T05:45:19,742 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T05:45:19,750 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T05:45:19,752 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T05:45:19,761 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T05:45:19,764 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T05:45:19,777 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T05:45:19,787 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:19,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:19,787 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:19,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,788 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4da2d47fc9ac,45173,1732686319178, sessionid=0x1017ad309e40000, setting cluster-up flag (Was=false) 2024-11-27T05:45:19,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,809 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,840 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T05:45:19,842 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:19,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,861 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:19,893 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T05:45:19,894 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:19,895 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T05:45:19,896 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:19,897 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T05:45:19,897 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T05:45:19,897 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4da2d47fc9ac,45173,1732686319178 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T05:45:19,899 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:19,899 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:19,899 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:19,900 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=5, maxPoolSize=5 2024-11-27T05:45:19,900 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4da2d47fc9ac:0, corePoolSize=10, maxPoolSize=10 2024-11-27T05:45:19,900 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:19,900 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=2, maxPoolSize=2 2024-11-27T05:45:19,900 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:19,901 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732686349901 2024-11-27T05:45:19,901 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T05:45:19,901 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T05:45:19,901 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T05:45:19,901 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T05:45:19,902 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T05:45:19,902 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T05:45:19,902 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:19,902 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:19,902 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T05:45:19,902 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T05:45:19,902 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T05:45:19,902 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T05:45:19,902 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,903 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T05:45:19,904 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T05:45:19,904 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T05:45:19,912 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.large.0-1732686319904,5,FailOnTimeoutGroup] 2024-11-27T05:45:19,916 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.small.0-1732686319912,5,FailOnTimeoutGroup] 2024-11-27T05:45:19,916 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:19,917 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T05:45:19,917 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:19,917 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:19,932 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T05:45:19,932 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:19,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:19,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T05:45:19,945 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T05:45:19,945 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:19,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T05:45:19,949 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T05:45:19,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:19,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T05:45:19,953 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T05:45:19,953 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:19,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T05:45:19,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T05:45:19,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:19,957 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:19,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T05:45:19,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:19,958 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:19,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T05:45:19,959 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T05:45:19,959 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T05:45:19,960 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T05:45:19,966 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T05:45:19,966 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68729174, jitterRate=0.024144500494003296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T05:45:19,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732686319940Initializing all the Stores at 1732686319940Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686319940Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686319940Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686319940Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686319940Cleaning up temporary data from old regions at 1732686319959 (+19 ms)Region opened successfully at 1732686319967 (+8 ms) 2024-11-27T05:45:19,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T05:45:19,967 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T05:45:19,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T05:45:19,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T05:45:19,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T05:45:19,968 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:19,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732686319967Disabling compacts and flushes for region at 1732686319967Disabling writes for close at 1732686319967Writing region close event to WAL at 1732686319968 (+1 ms)Closed at 1732686319968 2024-11-27T05:45:19,970 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:19,970 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T05:45:19,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T05:45:19,971 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T05:45:19,973 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T05:45:19,992 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(746): ClusterId : 0968198c-6dbe-45d3-8c0d-7ebbe4604d27 2024-11-27T05:45:19,992 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(746): ClusterId : 0968198c-6dbe-45d3-8c0d-7ebbe4604d27 2024-11-27T05:45:19,992 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T05:45:19,992 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T05:45:20,004 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T05:45:20,004 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T05:45:20,004 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T05:45:20,004 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T05:45:20,015 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T05:45:20,015 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T05:45:20,016 DEBUG [RS:0;4da2d47fc9ac:37641 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65fc57dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4da2d47fc9ac/172.17.0.2:0 2024-11-27T05:45:20,016 DEBUG [RS:1;4da2d47fc9ac:41581 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f58f137, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4da2d47fc9ac/172.17.0.2:0 2024-11-27T05:45:20,033 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4da2d47fc9ac:41581 2024-11-27T05:45:20,033 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T05:45:20,033 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T05:45:20,033 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T05:45:20,034 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(2659): reportForDuty to master=4da2d47fc9ac,45173,1732686319178 with port=41581, startcode=1732686319410 2024-11-27T05:45:20,035 DEBUG [RS:1;4da2d47fc9ac:41581 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T05:45:20,039 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37521, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T05:45:20,040 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45173 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,040 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45173 {}] master.ServerManager(517): Registering regionserver=4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,040 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4da2d47fc9ac:37641 2024-11-27T05:45:20,041 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T05:45:20,041 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T05:45:20,041 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T05:45:20,042 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:20,042 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(2659): reportForDuty to master=4da2d47fc9ac,45173,1732686319178 with port=37641, startcode=1732686319363 2024-11-27T05:45:20,042 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T05:45:20,042 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T05:45:20,042 DEBUG [RS:0;4da2d47fc9ac:37641 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T05:45:20,046 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42217, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T05:45:20,047 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45173 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:20,047 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45173 {}] master.ServerManager(517): Registering regionserver=4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:20,049 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:20,049 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T05:45:20,049 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T05:45:20,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T05:45:20,087 DEBUG [RS:1;4da2d47fc9ac:41581 {}] zookeeper.ZKUtil(111): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,087 WARN [RS:1;4da2d47fc9ac:41581 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T05:45:20,087 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4da2d47fc9ac,37641,1732686319363] 2024-11-27T05:45:20,087 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4da2d47fc9ac,41581,1732686319410] 2024-11-27T05:45:20,087 INFO [RS:1;4da2d47fc9ac:41581 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:20,087 DEBUG [RS:0;4da2d47fc9ac:37641 {}] zookeeper.ZKUtil(111): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:20,087 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,087 WARN [RS:0;4da2d47fc9ac:37641 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T05:45:20,088 INFO [RS:0;4da2d47fc9ac:37641 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:20,088 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:20,093 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T05:45:20,093 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T05:45:20,095 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T05:45:20,095 INFO [RS:1;4da2d47fc9ac:41581 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T05:45:20,096 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,096 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T05:45:20,096 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T05:45:20,097 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=2, maxPoolSize=2 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,097 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,098 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,098 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,098 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,098 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:20,098 DEBUG [RS:1;4da2d47fc9ac:41581 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:20,098 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T05:45:20,100 INFO [RS:0;4da2d47fc9ac:37641 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T05:45:20,100 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,101 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,101 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,101 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,101 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,101 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T05:45:20,101 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,101 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,41581,1732686319410-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T05:45:20,101 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T05:45:20,102 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=2, maxPoolSize=2 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,102 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,103 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,103 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,103 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:20,103 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:20,103 DEBUG [RS:0;4da2d47fc9ac:37641 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:20,103 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,104 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,104 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,104 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,104 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,104 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,37641,1732686319363-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T05:45:20,121 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T05:45:20,122 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,41581,1732686319410-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,122 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,122 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.Replication(171): 4da2d47fc9ac,41581,1732686319410 started 2024-11-27T05:45:20,123 WARN [4da2d47fc9ac:45173 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-27T05:45:20,129 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T05:45:20,130 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,37641,1732686319363-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,130 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,130 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.Replication(171): 4da2d47fc9ac,37641,1732686319363 started 2024-11-27T05:45:20,143 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,143 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1482): Serving as 4da2d47fc9ac,41581,1732686319410, RpcServer on 4da2d47fc9ac/172.17.0.2:41581, sessionid=0x1017ad309e40002 2024-11-27T05:45:20,144 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T05:45:20,144 DEBUG [RS:1;4da2d47fc9ac:41581 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,144 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,41581,1732686319410' 2024-11-27T05:45:20,144 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T05:45:20,147 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T05:45:20,148 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T05:45:20,148 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T05:45:20,148 DEBUG [RS:1;4da2d47fc9ac:41581 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,148 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,41581,1732686319410' 2024-11-27T05:45:20,148 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T05:45:20,149 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T05:45:20,149 DEBUG [RS:1;4da2d47fc9ac:41581 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T05:45:20,149 INFO [RS:1;4da2d47fc9ac:41581 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T05:45:20,149 INFO [RS:1;4da2d47fc9ac:41581 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T05:45:20,150 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,151 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1482): Serving as 4da2d47fc9ac,37641,1732686319363, RpcServer on 4da2d47fc9ac/172.17.0.2:37641, sessionid=0x1017ad309e40001 2024-11-27T05:45:20,151 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T05:45:20,151 DEBUG [RS:0;4da2d47fc9ac:37641 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:20,151 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,37641,1732686319363' 2024-11-27T05:45:20,151 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T05:45:20,152 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T05:45:20,152 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T05:45:20,152 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T05:45:20,152 DEBUG [RS:0;4da2d47fc9ac:37641 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:20,152 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,37641,1732686319363' 2024-11-27T05:45:20,152 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T05:45:20,153 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T05:45:20,154 DEBUG [RS:0;4da2d47fc9ac:37641 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T05:45:20,154 INFO [RS:0;4da2d47fc9ac:37641 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T05:45:20,154 INFO [RS:0;4da2d47fc9ac:37641 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T05:45:20,251 INFO [RS:1;4da2d47fc9ac:41581 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T05:45:20,252 INFO [RS:1;4da2d47fc9ac:41581 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C41581%2C1732686319410, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:20,255 INFO [RS:0;4da2d47fc9ac:37641 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T05:45:20,256 INFO [RS:0;4da2d47fc9ac:37641 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C37641%2C1732686319363, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37641,1732686319363, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:20,277 INFO [RS:1;4da2d47fc9ac:41581 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410/4da2d47fc9ac%2C41581%2C1732686319410.1732686320255 2024-11-27T05:45:20,277 DEBUG [RS:1;4da2d47fc9ac:41581 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:20,277 INFO [RS:0;4da2d47fc9ac:37641 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37641,1732686319363/4da2d47fc9ac%2C37641%2C1732686319363.1732686320257 2024-11-27T05:45:20,278 DEBUG [RS:0;4da2d47fc9ac:37641 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:20,373 DEBUG [4da2d47fc9ac:45173 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-27T05:45:20,374 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(204): Hosts are {4da2d47fc9ac=0} racks are {/default-rack=0} 2024-11-27T05:45:20,375 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T05:45:20,375 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T05:45:20,375 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T05:45:20,375 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T05:45:20,375 INFO [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T05:45:20,375 INFO [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T05:45:20,375 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T05:45:20,376 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,377 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4da2d47fc9ac,41581,1732686319410, state=OPENING 2024-11-27T05:45:20,403 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T05:45:20,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:20,414 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:20,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:20,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:20,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:20,415 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:20,416 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T05:45:20,416 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,41581,1732686319410}] 2024-11-27T05:45:20,569 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T05:45:20,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51269, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T05:45:20,575 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T05:45:20,576 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:20,576 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T05:45:20,578 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C41581%2C1732686319410.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:20,603 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410/4da2d47fc9ac%2C41581%2C1732686319410.meta.1732686320579.meta 2024-11-27T05:45:20,605 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:20,605 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:20,606 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T05:45:20,606 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T05:45:20,606 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T05:45:20,607 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T05:45:20,607 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:20,607 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T05:45:20,607 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T05:45:20,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T05:45:20,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T05:45:20,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:20,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:20,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T05:45:20,610 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T05:45:20,610 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:20,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:20,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T05:45:20,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T05:45:20,611 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:20,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:20,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T05:45:20,612 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T05:45:20,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:20,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:20,613 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T05:45:20,614 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:20,614 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:20,615 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T05:45:20,615 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T05:45:20,615 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T05:45:20,616 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T05:45:20,616 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70915175, jitterRate=0.056718453764915466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T05:45:20,616 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T05:45:20,617 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732686320607Writing region info on filesystem at 1732686320607Initializing all the Stores at 1732686320608 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686320608Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686320608Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686320608Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686320608Cleaning up temporary data from old regions at 1732686320615 (+7 ms)Running coprocessor post-open hooks at 1732686320616 (+1 ms)Region opened successfully at 1732686320617 (+1 ms) 2024-11-27T05:45:20,618 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732686320569 2024-11-27T05:45:20,621 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T05:45:20,621 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T05:45:20,621 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,622 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4da2d47fc9ac,41581,1732686319410, state=OPEN 2024-11-27T05:45:20,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:20,635 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,635 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:20,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:20,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:20,635 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:20,636 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:20,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T05:45:20,639 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,41581,1732686319410 in 219 msec 2024-11-27T05:45:20,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T05:45:20,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 669 msec 2024-11-27T05:45:20,643 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T05:45:20,643 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T05:45:20,658 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T05:45:20,659 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,41581,1732686319410, seqNum=-1] 2024-11-27T05:45:20,659 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T05:45:20,661 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36973, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T05:45:20,669 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 772 msec 2024-11-27T05:45:20,669 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732686320669, completionTime=-1 2024-11-27T05:45:20,669 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-27T05:45:20,669 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T05:45:20,673 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-27T05:45:20,673 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732686380673 2024-11-27T05:45:20,673 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732686440673 2024-11-27T05:45:20,673 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-11-27T05:45:20,674 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45173,1732686319178-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,674 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45173,1732686319178-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,674 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45173,1732686319178-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,674 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4da2d47fc9ac:45173, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,675 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,676 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,685 DEBUG [master/4da2d47fc9ac:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T05:45:20,691 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.151sec 2024-11-27T05:45:20,692 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T05:45:20,692 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T05:45:20,692 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T05:45:20,692 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T05:45:20,692 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T05:45:20,692 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45173,1732686319178-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T05:45:20,693 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45173,1732686319178-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T05:45:20,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@765880bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T05:45:20,693 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4da2d47fc9ac,45173,-1 for getting cluster id 2024-11-27T05:45:20,693 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T05:45:20,694 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T05:45:20,694 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T05:45:20,694 INFO [master/4da2d47fc9ac:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45173,1732686319178-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:20,697 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0968198c-6dbe-45d3-8c0d-7ebbe4604d27' 2024-11-27T05:45:20,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T05:45:20,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0968198c-6dbe-45d3-8c0d-7ebbe4604d27" 2024-11-27T05:45:20,699 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fb8d28c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T05:45:20,699 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4da2d47fc9ac,45173,-1] 2024-11-27T05:45:20,699 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T05:45:20,699 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:20,702 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T05:45:20,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37eabeac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T05:45:20,704 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T05:45:20,706 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,41581,1732686319410, seqNum=-1] 2024-11-27T05:45:20,707 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T05:45:20,709 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T05:45:20,712 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:20,713 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T05:45:20,715 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:20,715 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@326eea32 2024-11-27T05:45:20,715 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T05:45:20,719 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T05:45:20,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45173 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732686320712', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T05:45:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45173 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 2024-11-27T05:45:20,722 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T05:45:20,723 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:20,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45173 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732686320712" procId is: 4 2024-11-27T05:45:20,724 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T05:45:20,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:20,737 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1ec7e90d16bebe290c8a7516dc072b63, NAME => 'testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732686320712', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:20,749 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:20,749 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(1722): Closing 1ec7e90d16bebe290c8a7516dc072b63, disabling compactions & flushes 2024-11-27T05:45:20,749 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:20,749 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:20,749 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. after waiting 0 ms 2024-11-27T05:45:20,749 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:20,749 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:20,749 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732686320712-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1ec7e90d16bebe290c8a7516dc072b63: Waiting for close lock at 1732686320749Disabling compacts and flushes for region at 1732686320749Disabling writes for close at 1732686320749Writing region close event to WAL at 1732686320749Closed at 1732686320749 2024-11-27T05:45:20,751 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T05:45:20,751 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732686320751"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732686320751"}]},"ts":"1732686320751"} 2024-11-27T05:45:20,755 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T05:45:20,757 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T05:45:20,757 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732686320712","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732686320757"}]},"ts":"1732686320757"} 2024-11-27T05:45:20,761 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732686320712, state=ENABLING in hbase:meta 2024-11-27T05:45:20,761 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4da2d47fc9ac=0} racks are {/default-rack=0} 2024-11-27T05:45:20,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T05:45:20,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T05:45:20,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T05:45:20,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T05:45:20,762 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T05:45:20,762 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T05:45:20,762 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T05:45:20,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN}] 2024-11-27T05:45:20,765 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN 2024-11-27T05:45:20,766 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN; state=OFFLINE, location=4da2d47fc9ac,41581,1732686319410; forceNewPlan=false, retain=false 2024-11-27T05:45:20,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:20,917 INFO [4da2d47fc9ac:45173 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-27T05:45:20,917 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1ec7e90d16bebe290c8a7516dc072b63, regionState=OPENING, regionLocation=4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:20,921 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN because future has completed 2024-11-27T05:45:20,922 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ec7e90d16bebe290c8a7516dc072b63, server=4da2d47fc9ac,41581,1732686319410}] 2024-11-27T05:45:21,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:21,065 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T05:45:21,079 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:21,080 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1ec7e90d16bebe290c8a7516dc072b63, NAME => 'testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63.', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:21,080 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732686320712 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,080 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:21,081 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,081 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,082 INFO [StoreOpener-1ec7e90d16bebe290c8a7516dc072b63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,089 INFO [StoreOpener-1ec7e90d16bebe290c8a7516dc072b63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1ec7e90d16bebe290c8a7516dc072b63 columnFamilyName family 2024-11-27T05:45:21,089 DEBUG [StoreOpener-1ec7e90d16bebe290c8a7516dc072b63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:21,089 INFO [StoreOpener-1ec7e90d16bebe290c8a7516dc072b63-1 {}] regionserver.HStore(327): Store=1ec7e90d16bebe290c8a7516dc072b63/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T05:45:21,090 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,091 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,091 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,091 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,091 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,092 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,133 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T05:45:21,134 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1ec7e90d16bebe290c8a7516dc072b63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69812423, jitterRate=0.040286168456077576}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-27T05:45:21,134 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:21,135 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1ec7e90d16bebe290c8a7516dc072b63: Running coprocessor pre-open hook at 1732686321081Writing region info on filesystem at 1732686321081Initializing all the Stores at 1732686321081Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686321081Cleaning up temporary data from old regions at 1732686321091 (+10 ms)Running coprocessor post-open hooks at 1732686321134 (+43 ms)Region opened successfully at 1732686321135 (+1 ms) 2024-11-27T05:45:21,136 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63., pid=6, masterSystemTime=1732686321075 2024-11-27T05:45:21,139 DEBUG [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:21,139 INFO [RS_OPEN_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:21,140 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1ec7e90d16bebe290c8a7516dc072b63, regionState=OPEN, openSeqNum=2, regionLocation=4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:21,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1ec7e90d16bebe290c8a7516dc072b63, server=4da2d47fc9ac,41581,1732686319410 because future has completed 2024-11-27T05:45:21,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T05:45:21,150 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1ec7e90d16bebe290c8a7516dc072b63, server=4da2d47fc9ac,41581,1732686319410 in 225 msec 2024-11-27T05:45:21,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T05:45:21,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN in 387 msec 2024-11-27T05:45:21,154 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T05:45:21,154 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732686320712","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732686321154"}]},"ts":"1732686321154"} 2024-11-27T05:45:21,158 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732686320712, state=ENABLED in hbase:meta 2024-11-27T05:45:21,159 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T05:45:21,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732686320712 in 439 msec 2024-11-27T05:45:21,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45173 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T05:45:21,366 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732686320712 completed 2024-11-27T05:45:21,366 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732686320712 get assigned. Timeout = 60000ms 2024-11-27T05:45:21,366 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T05:45:21,371 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732686320712 assigned to meta. Checking AM states. 2024-11-27T05:45:21,372 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-27T05:45:21,372 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732686320712 assigned. 2024-11-27T05:45:21,376 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732686320712', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63., hostname=4da2d47fc9ac,41581,1732686319410, seqNum=2] 2024-11-27T05:45:21,382 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1ec7e90d16bebe290c8a7516dc072b63 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-27T05:45:21,394 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5464ebce80144d988f4338119c18dd2b 2024-11-27T05:45:21,414 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/.tmp/family/5464ebce80144d988f4338119c18dd2b is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-27T05:45:21,415 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/.tmp/family/5464ebce80144d988f4338119c18dd2b 2024-11-27T05:45:21,419 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/.tmp/family/5464ebce80144d988f4338119c18dd2b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b 2024-11-27T05:45:21,421 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b, delay=922 ms 2024-11-27T05:45:21,423 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b, entries=2, sequenceid=6, filesize=4.9 K 2024-11-27T05:45:21,425 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 1ec7e90d16bebe290c8a7516dc072b63 in 43ms, sequenceid=6, compaction requested=false 2024-11-27T05:45:21,425 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732686320712' 2024-11-27T05:45:21,426 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1ec7e90d16bebe290c8a7516dc072b63: 2024-11-27T05:45:21,993 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T05:45:21,994 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T05:45:21,997 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732686320712 2024-11-27T05:45:21,997 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732686320712 Metrics about Tables on a single HBase RegionServer 2024-11-27T05:45:21,999 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T05:45:21,999 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T05:45:22,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T05:45:22,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T05:45:22,345 DEBUG [hfile-prefetch-1732686316911 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5464ebce80144d988f4338119c18dd2b_0. 2024-11-27T05:45:22,345 DEBUG [hfile-prefetch-1732686316911 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 5464ebce80144d988f4338119c18dd2b_117. 2024-11-27T05:45:22,346 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b, with total blocks 2, and data blocks 1 2024-11-27T05:45:22,346 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-27T05:45:22,346 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-27T05:45:22,346 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b. Will sleep 100ms and try the verification again. 2024-11-27T05:45:22,426 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;4da2d47fc9ac:41581,5,FailOnTimeoutGroup] 2024-11-27T05:45:22,427 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4da2d47fc9ac,41581,1732686319410' ***** 2024-11-27T05:45:22,427 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-27T05:45:22,427 INFO [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T05:45:22,437 INFO [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence 2024-11-27T05:45:22,437 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T05:45:22,437 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T05:45:22,437 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T05:45:22,437 INFO [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T05:45:22,437 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T05:45:22,446 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b, with total blocks 2, and data blocks 1 2024-11-27T05:45:22,447 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-27T05:45:22,447 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2293): found block for file 5464ebce80144d988f4338119c18dd2b in the backing map. Acquiring read lock for offset 0 2024-11-27T05:45:22,447 DEBUG [hfile-prefetch-1732686316911 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/family/5464ebce80144d988f4338119c18dd2b has now been fully cached. 2024-11-27T05:45:22,447 DEBUG [hfile-prefetch-1732686316911 {}] hfile.PrefetchExecutor(121): Prefetch completed for 5464ebce80144d988f4338119c18dd2b 2024-11-27T05:45:22,447 DEBUG [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1487): Thread RS:1;4da2d47fc9ac:41581 started persisting bucket cache to file 2024-11-27T05:45:22,447 DEBUG [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:22,447 DEBUG [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-27T05:45:22,457 DEBUG [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-27T05:45:22,458 DEBUG [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732686322447 2024-11-27T05:45:22,458 DEBUG [RS:1;4da2d47fc9ac:41581 {}] bucket.BucketCache(1509): Thread RS:1;4da2d47fc9ac:41581 finished persisting bucket cache to file, renaming 2024-11-27T05:45:22,458 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T05:45:22,458 INFO [RS:1;4da2d47fc9ac:41581 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T05:45:22,458 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T05:45:22,458 INFO [RS:1;4da2d47fc9ac:41581 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T05:45:22,458 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(3091): Received CLOSE for 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:22,459 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(959): stopping server 4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:22,459 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T05:45:22,459 INFO [RS:1;4da2d47fc9ac:41581 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4da2d47fc9ac:41581. 2024-11-27T05:45:22,459 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T05:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1ec7e90d16bebe290c8a7516dc072b63, disabling compactions & flushes 2024-11-27T05:45:22,459 DEBUG [RS:1;4da2d47fc9ac:41581 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:22,459 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:22,459 DEBUG [RS:1;4da2d47fc9ac:41581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. after waiting 0 ms 2024-11-27T05:45:22,459 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T05:45:22,459 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:22,459 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T05:45:22,459 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T05:45:22,459 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T05:45:22,460 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-27T05:45:22,460 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1325): Online Regions={1ec7e90d16bebe290c8a7516dc072b63=testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63., 1588230740=hbase:meta,,1.1588230740} 2024-11-27T05:45:22,460 DEBUG [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 1ec7e90d16bebe290c8a7516dc072b63 2024-11-27T05:45:22,460 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T05:45:22,460 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T05:45:22,460 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T05:45:22,460 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T05:45:22,460 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T05:45:22,460 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T05:45:22,460 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-27T05:45:22,471 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/default/testBlockEvictionOnGracefulStop1732686320712/1ec7e90d16bebe290c8a7516dc072b63/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-27T05:45:22,471 INFO [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:22,471 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1ec7e90d16bebe290c8a7516dc072b63: Waiting for close lock at 1732686322459Running coprocessor pre-close hooks at 1732686322459Disabling compacts and flushes for region at 1732686322459Disabling writes for close at 1732686322459Writing region close event to WAL at 1732686322460 (+1 ms)Running coprocessor post-close hooks at 1732686322471 (+11 ms)Closed at 1732686322471 2024-11-27T05:45:22,472 DEBUG [RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63. 2024-11-27T05:45:22,477 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f9e203a048ff40849f85d7784b74c182 2024-11-27T05:45:22,488 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-27T05:45:22,488 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-27T05:45:22,488 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:22,488 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T05:45:22,491 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/info/f9e203a048ff40849f85d7784b74c182 is 209, key is testBlockEvictionOnGracefulStop1732686320712,,1732686320719.1ec7e90d16bebe290c8a7516dc072b63./info:regioninfo/1732686321139/Put/seqid=0 2024-11-27T05:45:22,491 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/info/f9e203a048ff40849f85d7784b74c182 2024-11-27T05:45:22,495 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T05:45:22,495 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732686322488 2024-11-27T05:45:22,495 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-27T05:45:22,503 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ec745a2ae36c45bbb2cb2043eeebf358 2024-11-27T05:45:22,506 INFO [regionserver/4da2d47fc9ac:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:22,516 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/ns/ec745a2ae36c45bbb2cb2043eeebf358 is 43, key is default/ns:d/1732686320662/Put/seqid=0 2024-11-27T05:45:22,517 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/ns/ec745a2ae36c45bbb2cb2043eeebf358 2024-11-27T05:45:22,527 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dfe13dfb495845c2ad11fc6c3c9547be 2024-11-27T05:45:22,545 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/table/dfe13dfb495845c2ad11fc6c3c9547be is 80, key is testBlockEvictionOnGracefulStop1732686320712/table:state/1732686321154/Put/seqid=0 2024-11-27T05:45:22,546 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/table/dfe13dfb495845c2ad11fc6c3c9547be 2024-11-27T05:45:22,550 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/info/f9e203a048ff40849f85d7784b74c182 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/f9e203a048ff40849f85d7784b74c182 2024-11-27T05:45:22,552 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/f9e203a048ff40849f85d7784b74c182, delay=990 ms 2024-11-27T05:45:22,554 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/f9e203a048ff40849f85d7784b74c182, entries=10, sequenceid=11, filesize=7.0 K 2024-11-27T05:45:22,554 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/ns/ec745a2ae36c45bbb2cb2043eeebf358 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/ec745a2ae36c45bbb2cb2043eeebf358 2024-11-27T05:45:22,556 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/ec745a2ae36c45bbb2cb2043eeebf358, delay=944 ms 2024-11-27T05:45:22,558 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/ec745a2ae36c45bbb2cb2043eeebf358, entries=2, sequenceid=11, filesize=5.0 K 2024-11-27T05:45:22,558 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/.tmp/table/dfe13dfb495845c2ad11fc6c3c9547be as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/dfe13dfb495845c2ad11fc6c3c9547be 2024-11-27T05:45:22,560 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/dfe13dfb495845c2ad11fc6c3c9547be, delay=954 ms 2024-11-27T05:45:22,561 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/dfe13dfb495845c2ad11fc6c3c9547be, entries=2, sequenceid=11, filesize=5.3 K 2024-11-27T05:45:22,562 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=11, compaction requested=false 2024-11-27T05:45:22,563 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/f9e203a048ff40849f85d7784b74c182 2024-11-27T05:45:22,564 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/ec745a2ae36c45bbb2cb2043eeebf358 2024-11-27T05:45:22,565 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/dfe13dfb495845c2ad11fc6c3c9547be 2024-11-27T05:45:22,569 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-27T05:45:22,569 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T05:45:22,570 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:22,570 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732686322460Running coprocessor pre-close hooks at 1732686322460Disabling compacts and flushes for region at 1732686322460Disabling writes for close at 1732686322460Obtaining lock to block concurrent updates at 1732686322461 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732686322461Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732686322461Flushing stores of hbase:meta,,1.1588230740 at 1732686322462 (+1 ms)Flushing 1588230740/info: creating writer at 1732686322462Flushing 1588230740/info: appending metadata at 1732686322490 (+28 ms)Flushing 1588230740/info: closing flushed file at 1732686322490Flushing 1588230740/ns: creating writer at 1732686322494 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732686322516 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1732686322516Flushing 1588230740/table: creating writer at 1732686322519 (+3 ms)Flushing 1588230740/table: appending metadata at 1732686322544 (+25 ms)Flushing 1588230740/table: closing flushed file at 1732686322545 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1835bd8c: reopening flushed file at 1732686322550 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f2a5ee8: reopening flushed file at 1732686322554 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@45fc4153: reopening flushed file at 1732686322558 (+4 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=11, compaction requested=false at 1732686322562 (+4 ms)Writing region close event to WAL at 1732686322565 (+3 ms)Running coprocessor post-close hooks at 1732686322569 (+4 ms)Closed at 1732686322569 2024-11-27T05:45:22,570 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:22,660 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(976): stopping server 4da2d47fc9ac,41581,1732686319410; all regions closed. 2024-11-27T05:45:22,850 DEBUG [RS:1;4da2d47fc9ac:41581 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:22,851 INFO [RS:1;4da2d47fc9ac:41581 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C41581%2C1732686319410.meta:.meta(num 1732686320579) 2024-11-27T05:45:22,856 DEBUG [RS:1;4da2d47fc9ac:41581 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:22,857 INFO [RS:1;4da2d47fc9ac:41581 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C41581%2C1732686319410:(num 1732686320255) 2024-11-27T05:45:22,857 DEBUG [RS:1;4da2d47fc9ac:41581 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:22,857 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:22,857 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T05:45:22,858 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.ChoreService(370): Chore service for: regionserver/4da2d47fc9ac:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T05:45:22,859 INFO [regionserver/4da2d47fc9ac:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T05:45:22,859 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T05:45:22,860 INFO [RS:1;4da2d47fc9ac:41581 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41581 2024-11-27T05:45:22,949 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4da2d47fc9ac:0 server-side Connection retries=45 2024-11-27T05:45:22,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:22,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:22,949 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T05:45:22,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T05:45:22,949 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T05:45:22,949 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T05:45:22,949 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T05:45:22,950 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45831 2024-11-27T05:45:22,952 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45831 connecting to ZooKeeper ensemble=127.0.0.1:56667 2024-11-27T05:45:22,952 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:22,952 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:23,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T05:45:23,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4da2d47fc9ac,41581,1732686319410 2024-11-27T05:45:23,028 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T05:45:23,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:458310x0, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T05:45:23,072 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45831-0x1017ad309e40003 connected 2024-11-27T05:45:23,114 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4da2d47fc9ac,41581,1732686319410] 2024-11-27T05:45:23,114 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-27T05:45:23,114 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-27T05:45:23,116 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T05:45:23,117 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.cache 2024-11-27T05:45:23,118 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-27T05:45:23,118 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-27T05:45:23,118 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-27T05:45:23,119 INFO [Thread-207 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-27T05:45:23,119 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-27T05:45:23,120 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T05:45:23,121 INFO [Thread-207 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-27T05:45:23,122 INFO [Thread-207 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-27T05:45:23,127 DEBUG [Thread-207 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-27T05:45:23,128 INFO [Thread-207 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-27T05:45:23,128 INFO [Thread-207 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-27T05:45:23,128 INFO [Thread-207 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence is complete. 2024-11-27T05:45:23,145 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4da2d47fc9ac,41581,1732686319410 already deleted, retry=false 2024-11-27T05:45:23,145 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T05:45:23,145 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 4da2d47fc9ac,41581,1732686319410 on 4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:23,148 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T05:45:23,149 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45831 2024-11-27T05:45:23,150 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45831 2024-11-27T05:45:23,150 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45831 2024-11-27T05:45:23,152 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45831 2024-11-27T05:45:23,153 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45831 2024-11-27T05:45:23,153 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 4da2d47fc9ac,41581,1732686319410, splitWal=true, meta=true 2024-11-27T05:45:23,155 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(746): ClusterId : 0968198c-6dbe-45d3-8c0d-7ebbe4604d27 2024-11-27T05:45:23,155 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T05:45:23,156 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 4da2d47fc9ac,41581,1732686319410, splitWal=true, meta=true 2024-11-27T05:45:23,157 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 4da2d47fc9ac,41581,1732686319410 (carryingMeta=true) 4da2d47fc9ac,41581,1732686319410/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@ebf7c59[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-27T05:45:23,158 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 4da2d47fc9ac,41581,1732686319410, splitWal=true, meta=true, isMeta: true 2024-11-27T05:45:23,159 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410-splitting 2024-11-27T05:45:23,159 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410-splitting dir is empty, no logs to split. 2024-11-27T05:45:23,159 INFO [PEWorker-2 {}] master.SplitWALManager(105): 4da2d47fc9ac,41581,1732686319410 WAL count=0, meta=true 2024-11-27T05:45:23,162 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410-splitting dir is empty, no logs to split. 2024-11-27T05:45:23,162 INFO [PEWorker-2 {}] master.SplitWALManager(105): 4da2d47fc9ac,41581,1732686319410 WAL count=0, meta=true 2024-11-27T05:45:23,162 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 4da2d47fc9ac,41581,1732686319410 WAL splitting is done? wals=0, meta=true 2024-11-27T05:45:23,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T05:45:23,164 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T05:45:23,165 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T05:45:23,178 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T05:45:23,178 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T05:45:23,199 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T05:45:23,199 DEBUG [RS:2;4da2d47fc9ac:45831 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23eee332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4da2d47fc9ac/172.17.0.2:0 2024-11-27T05:45:23,210 DEBUG [RS:2;4da2d47fc9ac:45831 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;4da2d47fc9ac:45831 2024-11-27T05:45:23,210 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T05:45:23,210 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T05:45:23,210 DEBUG [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T05:45:23,211 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(2659): reportForDuty to master=4da2d47fc9ac,45173,1732686319178 with port=45831, startcode=1732686322949 2024-11-27T05:45:23,212 DEBUG [RS:2;4da2d47fc9ac:45831 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T05:45:23,213 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49819, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T05:45:23,214 INFO [RS:1;4da2d47fc9ac:41581 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T05:45:23,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:23,214 INFO [RS:1;4da2d47fc9ac:41581 {}] regionserver.HRegionServer(1031): Exiting; stopping=4da2d47fc9ac,41581,1732686319410; zookeeper connection closed. 2024-11-27T05:45:23,214 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41581-0x1017ad309e40002, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:23,214 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45173 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,214 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45173 {}] master.ServerManager(517): Registering regionserver=4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,214 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1be6560c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1be6560c 2024-11-27T05:45:23,215 DEBUG [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93 2024-11-27T05:45:23,215 DEBUG [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-27T05:45:23,215 DEBUG [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T05:45:23,229 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T05:45:23,230 DEBUG [RS:2;4da2d47fc9ac:45831 {}] zookeeper.ZKUtil(111): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,230 WARN [RS:2;4da2d47fc9ac:45831 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T05:45:23,230 INFO [RS:2;4da2d47fc9ac:45831 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:23,230 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4da2d47fc9ac,45831,1732686322949] 2024-11-27T05:45:23,230 DEBUG [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,235 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T05:45:23,237 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T05:45:23,238 INFO [RS:2;4da2d47fc9ac:45831 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T05:45:23,238 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,238 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T05:45:23,239 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T05:45:23,239 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,239 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,239 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,239 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,239 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=2, maxPoolSize=2 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4da2d47fc9ac:0, corePoolSize=1, maxPoolSize=1 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:23,240 DEBUG [RS:2;4da2d47fc9ac:45831 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4da2d47fc9ac:0, corePoolSize=3, maxPoolSize=3 2024-11-27T05:45:23,240 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,241 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,241 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,241 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,241 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,241 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45831,1732686322949-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T05:45:23,258 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T05:45:23,258 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=4da2d47fc9ac,45831,1732686322949-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,258 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,258 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.Replication(171): 4da2d47fc9ac,45831,1732686322949 started 2024-11-27T05:45:23,274 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T05:45:23,274 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(1482): Serving as 4da2d47fc9ac,45831,1732686322949, RpcServer on 4da2d47fc9ac/172.17.0.2:45831, sessionid=0x1017ad309e40003 2024-11-27T05:45:23,274 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T05:45:23,275 DEBUG [RS:2;4da2d47fc9ac:45831 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,275 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,45831,1732686322949' 2024-11-27T05:45:23,275 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T05:45:23,275 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T05:45:23,276 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T05:45:23,276 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T05:45:23,276 DEBUG [RS:2;4da2d47fc9ac:45831 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,276 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4da2d47fc9ac,45831,1732686322949' 2024-11-27T05:45:23,276 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T05:45:23,277 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T05:45:23,277 DEBUG [RS:2;4da2d47fc9ac:45831 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T05:45:23,277 INFO [RS:2;4da2d47fc9ac:45831 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T05:45:23,277 INFO [RS:2;4da2d47fc9ac:45831 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T05:45:23,315 DEBUG [4da2d47fc9ac:45173 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T05:45:23,316 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:23,317 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4da2d47fc9ac,37641,1732686319363, state=OPENING 2024-11-27T05:45:23,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:23,329 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:23,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:23,329 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T05:45:23,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:23,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:23,329 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:23,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,37641,1732686319363}] 2024-11-27T05:45:23,378 INFO [RS:2;4da2d47fc9ac:45831 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T05:45:23,378 INFO [RS:2;4da2d47fc9ac:45831 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C45831%2C1732686322949, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,45831,1732686322949, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:23,396 INFO [RS:2;4da2d47fc9ac:45831 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,45831,1732686322949/4da2d47fc9ac%2C45831%2C1732686322949.1732686323379 2024-11-27T05:45:23,397 DEBUG [RS:2;4da2d47fc9ac:45831 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:23,482 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T05:45:23,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49655, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T05:45:23,487 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T05:45:23,487 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T05:45:23,487 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T05:45:23,489 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4da2d47fc9ac%2C37641%2C1732686319363.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37641,1732686319363, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs, maxLogs=55 2024-11-27T05:45:23,513 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,37641,1732686319363/4da2d47fc9ac%2C37641%2C1732686319363.meta.1732686323493.meta 2024-11-27T05:45:23,513 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-27T05:45:23,514 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T05:45:23,514 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T05:45:23,514 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T05:45:23,514 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T05:45:23,514 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T05:45:23,514 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T05:45:23,514 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T05:45:23,515 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T05:45:23,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-27T05:45:23,516 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T05:45:23,516 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:23,517 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/f9e203a048ff40849f85d7784b74c182, delay=927 ms 2024-11-27T05:45:23,519 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/f9e203a048ff40849f85d7784b74c182 2024-11-27T05:45:23,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:23,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-27T05:45:23,520 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T05:45:23,520 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:23,522 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/ec745a2ae36c45bbb2cb2043eeebf358, delay=968 ms 2024-11-27T05:45:23,523 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/ec745a2ae36c45bbb2cb2043eeebf358 2024-11-27T05:45:23,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:23,524 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-27T05:45:23,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T05:45:23,525 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:23,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:23,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-27T05:45:23,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T05:45:23,526 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T05:45:23,528 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/dfe13dfb495845c2ad11fc6c3c9547be, delay=981 ms 2024-11-27T05:45:23,530 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/dfe13dfb495845c2ad11fc6c3c9547be 2024-11-27T05:45:23,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T05:45:23,530 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T05:45:23,531 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:23,532 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740 2024-11-27T05:45:23,532 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T05:45:23,532 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T05:45:23,533 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T05:45:23,534 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T05:45:23,535 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72511950, jitterRate=0.08051225543022156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T05:45:23,535 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T05:45:23,536 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732686323515Writing region info on filesystem at 1732686323515Initializing all the Stores at 1732686323515Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686323515Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686323515Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732686323515Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732686323515Cleaning up temporary data from old regions at 1732686323532 (+17 ms)Running coprocessor post-open hooks at 1732686323535 (+3 ms)Region opened successfully at 1732686323536 (+1 ms) 2024-11-27T05:45:23,537 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732686323482 2024-11-27T05:45:23,540 DEBUG [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T05:45:23,540 INFO [RS_OPEN_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T05:45:23,541 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:23,542 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4da2d47fc9ac,37641,1732686319363, state=OPEN 2024-11-27T05:45:23,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:23,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:23,556 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T05:45:23,556 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:23,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:23,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:23,556 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T05:45:23,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-27T05:45:23,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4da2d47fc9ac,37641,1732686319363 in 227 msec 2024-11-27T05:45:23,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T05:45:23,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 395 msec 2024-11-27T05:45:23,561 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 4da2d47fc9ac,41581,1732686319410 had 2 regions 2024-11-27T05:45:23,562 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 4da2d47fc9ac,41581,1732686319410, splitWal=true, meta=true, isMeta: false 2024-11-27T05:45:23,562 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410-splitting dir is empty, no logs to split. 2024-11-27T05:45:23,562 INFO [PEWorker-3 {}] master.SplitWALManager(105): 4da2d47fc9ac,41581,1732686319410 WAL count=0, meta=false 2024-11-27T05:45:23,563 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/WALs/4da2d47fc9ac,41581,1732686319410-splitting dir is empty, no logs to split. 2024-11-27T05:45:23,563 INFO [PEWorker-3 {}] master.SplitWALManager(105): 4da2d47fc9ac,41581,1732686319410 WAL count=0, meta=false 2024-11-27T05:45:23,563 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 4da2d47fc9ac,41581,1732686319410 WAL splitting is done? wals=0, meta=false 2024-11-27T05:45:23,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN}] 2024-11-27T05:45:23,566 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN 2024-11-27T05:45:23,567 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-27T05:45:23,717 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(204): Hosts are {4da2d47fc9ac=0} racks are {/default-rack=0} 2024-11-27T05:45:23,718 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-27T05:45:23,718 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-27T05:45:23,718 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-27T05:45:23,718 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-27T05:45:23,718 INFO [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-27T05:45:23,718 INFO [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-27T05:45:23,718 DEBUG [4da2d47fc9ac:45173 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-27T05:45:23,718 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=1ec7e90d16bebe290c8a7516dc072b63, regionState=OPENING, regionLocation=4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:23,720 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 4da2d47fc9ac:41581 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4da2d47fc9ac/172.17.0.2:41581 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T05:45:23,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,41581,1732686319410, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,41581,1732686319410, seqNum=-1, error=java.net.ConnectException: Call to address=4da2d47fc9ac:41581 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4da2d47fc9ac/172.17.0.2:41581 2024-11-27T05:45:23,722 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,41581,1732686319410, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-27T05:45:23,723 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,41581,1732686319410, seqNum=-1 from cache 2024-11-27T05:45:23,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 4da2d47fc9ac:41581 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 4da2d47fc9ac/172.17.0.2:41581 2024-11-27T05:45:23,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T05:45:23,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T05:45:23,775 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:23,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:23,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:23,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T05:45:23,777 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T05:45:23,777 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=614442724, stopped=false 2024-11-27T05:45:23,777 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4da2d47fc9ac,45173,1732686319178 2024-11-27T05:45:23,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:23,798 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:23,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T05:45:23,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:23,798 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:23,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T05:45:23,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:23,798 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T05:45:23,798 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:23,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:23,799 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:23,799 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4da2d47fc9ac,37641,1732686319363' ***** 2024-11-27T05:45:23,799 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T05:45:23,799 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4da2d47fc9ac,45831,1732686322949' ***** 2024-11-27T05:45:23,799 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:23,799 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T05:45:23,799 INFO [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T05:45:23,799 INFO [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1738): Disabling cache 2024-11-27T05:45:23,799 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T05:45:23,799 INFO [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence 2024-11-27T05:45:23,799 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T05:45:23,799 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T05:45:23,799 INFO [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/15a09b2e-7466-d5ff-a97d-2c47e504a195/bucket.persistence 2024-11-27T05:45:23,799 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-27T05:45:23,799 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-27T05:45:23,799 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T05:45:23,799 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-27T05:45:23,799 INFO [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T05:45:23,799 INFO [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-27T05:45:23,800 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T05:45:23,800 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-27T05:45:23,800 DEBUG [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1487): Thread RS:2;4da2d47fc9ac:45831 started persisting bucket cache to file 2024-11-27T05:45:23,800 DEBUG [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:23,800 DEBUG [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T05:45:23,803 DEBUG [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T05:45:23,803 DEBUG [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732686323800 2024-11-27T05:45:23,803 DEBUG [RS:2;4da2d47fc9ac:45831 {}] bucket.BucketCache(1509): Thread RS:2;4da2d47fc9ac:45831 finished persisting bucket cache to file, renaming 2024-11-27T05:45:23,803 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T05:45:23,803 INFO [RS:2;4da2d47fc9ac:45831 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T05:45:23,803 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T05:45:23,803 INFO [RS:2;4da2d47fc9ac:45831 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T05:45:23,804 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(959): stopping server 4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,804 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T05:45:23,804 INFO [RS:2;4da2d47fc9ac:45831 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;4da2d47fc9ac:45831. 2024-11-27T05:45:23,804 DEBUG [RS:2;4da2d47fc9ac:45831 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:23,804 DEBUG [RS:2;4da2d47fc9ac:45831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:23,804 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(976): stopping server 4da2d47fc9ac,45831,1732686322949; all regions closed. 2024-11-27T05:45:23,806 DEBUG [RS:2;4da2d47fc9ac:45831 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:23,806 INFO [RS:2;4da2d47fc9ac:45831 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C45831%2C1732686322949:(num 1732686323379) 2024-11-27T05:45:23,806 DEBUG [RS:2;4da2d47fc9ac:45831 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:23,806 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:23,806 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T05:45:23,806 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.ChoreService(370): Chore service for: regionserver/4da2d47fc9ac:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T05:45:23,807 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T05:45:23,807 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T05:45:23,807 INFO [regionserver/4da2d47fc9ac:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T05:45:23,807 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T05:45:23,807 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T05:45:23,807 INFO [RS:2;4da2d47fc9ac:45831 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45831 2024-11-27T05:45:23,810 DEBUG [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1487): Thread RS:0;4da2d47fc9ac:37641 started persisting bucket cache to file 2024-11-27T05:45:23,810 DEBUG [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-27T05:45:23,810 DEBUG [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-27T05:45:23,810 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T05:45:23,811 INFO [regionserver/4da2d47fc9ac:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:23,813 DEBUG [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-27T05:45:23,813 DEBUG [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732686323810 2024-11-27T05:45:23,813 DEBUG [RS:0;4da2d47fc9ac:37641 {}] bucket.BucketCache(1509): Thread RS:0;4da2d47fc9ac:37641 finished persisting bucket cache to file, renaming 2024-11-27T05:45:23,833 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T05:45:23,833 INFO [RS:0;4da2d47fc9ac:37641 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T05:45:23,833 INFO [RS:0;4da2d47fc9ac:37641 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T05:45:23,833 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(959): stopping server 4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:23,833 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T05:45:23,833 INFO [RS:0;4da2d47fc9ac:37641 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4da2d47fc9ac:37641. 2024-11-27T05:45:23,833 DEBUG [RS:0;4da2d47fc9ac:37641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T05:45:23,833 DEBUG [RS:0;4da2d47fc9ac:37641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:23,834 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T05:45:23,834 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T05:45:23,834 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T05:45:23,834 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T05:45:23,834 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T05:45:23,834 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4da2d47fc9ac,37641,1732686319363, seqNum=-1] 2024-11-27T05:45:23,834 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T05:45:23,834 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T05:45:23,834 DEBUG [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T05:45:23,834 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-27T05:45:23,834 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T05:45:23,834 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T05:45:23,834 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T05:45:23,834 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T05:45:23,834 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T05:45:23,834 DEBUG [Async-Client-Retry-Timer-pool-0 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732686320712, region=1ec7e90d16bebe290c8a7516dc072b63, ASSIGN because future has completed 2024-11-27T05:45:23,835 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/info/f9e203a048ff40849f85d7784b74c182 2024-11-27T05:45:23,835 ERROR [Async-Client-Retry-Timer-pool-0 {}] util.FutureUtils(73): Unexpected error caught when processing CompletableFuture java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil$$Lambda$785/0x00007f92d07736e0@2a9c8661 rejected from java.util.concurrent.ThreadPoolExecutor@7640c[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 2] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.lambda$suspendIfNecessary$1(ProcedureFutureUtil.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:173) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T05:45:23,835 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/ns/ec745a2ae36c45bbb2cb2043eeebf358 2024-11-27T05:45:23,836 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/table/dfe13dfb495845c2ad11fc6c3c9547be 2024-11-27T05:45:23,835 ERROR [Async-Client-Retry-Timer-pool-0 {}] assignment.RegionStateStore(295): FAILED persisting region=1ec7e90d16bebe290c8a7516dc072b63 state=OPENING org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=4da2d47fc9ac:37641 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.mutate(ClientProtos.java:43836) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$mutate$0(RawAsyncTableImpl.java:180) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.call(ConnectionUtils.java:619) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.mutate(RawAsyncTableImpl.java:179) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.voidMutate(RawAsyncTableImpl.java:186) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$put$8(RawAsyncTableImpl.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 22 more 2024-11-27T05:45:23,837 ERROR [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3218): Master server abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-27T05:45:23,837 ERROR [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3223): ***** ABORTING master 4da2d47fc9ac,45173,1732686319178: FAILED persisting region=1ec7e90d16bebe290c8a7516dc072b63 state=OPENING ***** org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=4da2d47fc9ac:37641 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$Stub.mutate(ClientProtos.java:43836) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$mutate$0(RawAsyncTableImpl.java:180) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.call(ConnectionUtils.java:619) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.mutate(RawAsyncTableImpl.java:179) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.voidMutate(RawAsyncTableImpl.java:186) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.RawAsyncTableImpl.lambda$put$8(RawAsyncTableImpl.java:255) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.call(AsyncSingleRequestRpcRetryingCaller.java:84) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$doCall$7(AsyncSingleRequestRpcRetryingCaller.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture.uniWhenCompleteStage(CompletableFuture.java:887) ~[?:?] at java.util.concurrent.CompletableFuture.whenComplete(CompletableFuture.java:2325) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.addListener(FutureUtils.java:64) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.doCall(AsyncSingleRequestRpcRetryingCaller.java:108) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.lambda$tryScheduleRetry$1(AsyncRpcRetryingCaller.java:139) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:713) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:701) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 22 more 2024-11-27T05:45:23,838 INFO [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3321): ***** STOPPING master '4da2d47fc9ac,45173,1732686319178' ***** 2024-11-27T05:45:23,838 INFO [Async-Client-Retry-Timer-pool-0 {}] master.HMaster(3323): STOPPED: Stopped by Async-Client-Retry-Timer-pool-0 2024-11-27T05:45:23,838 INFO [M:0;4da2d47fc9ac:45173 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T05:45:23,838 INFO [M:0;4da2d47fc9ac:45173 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T05:45:23,838 DEBUG [M:0;4da2d47fc9ac:45173 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T05:45:23,838 DEBUG [M:0;4da2d47fc9ac:45173 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T05:45:23,838 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T05:45:23,839 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.small.0-1732686319912 {}] cleaner.HFileCleaner(306): Exit Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.small.0-1732686319912,5,FailOnTimeoutGroup] 2024-11-27T05:45:23,839 DEBUG [master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.large.0-1732686319904 {}] cleaner.HFileCleaner(306): Exit Thread[master/4da2d47fc9ac:0:becomeActiveMaster-HFileCleaner.large.0-1732686319904,5,FailOnTimeoutGroup] 2024-11-27T05:45:23,839 INFO [M:0;4da2d47fc9ac:45173 {}] hbase.ChoreService(370): Chore service for: master/4da2d47fc9ac:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T05:45:23,839 INFO [M:0;4da2d47fc9ac:45173 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T05:45:23,839 DEBUG [M:0;4da2d47fc9ac:45173 {}] master.HMaster(1795): Stopping service threads 2024-11-27T05:45:23,839 INFO [M:0;4da2d47fc9ac:45173 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T05:45:23,839 INFO [M:0;4da2d47fc9ac:45173 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T05:45:23,840 INFO [M:0;4da2d47fc9ac:45173 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=true 2024-11-27T05:45:23,840 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T05:45:23,842 INFO [regionserver/4da2d47fc9ac:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:23,844 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/data/hbase/meta/1588230740/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=14 2024-11-27T05:45:23,844 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T05:45:23,845 INFO [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:23,845 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732686323834Running coprocessor pre-close hooks at 1732686323834Disabling compacts and flushes for region at 1732686323834Disabling writes for close at 1732686323834Writing region close event to WAL at 1732686323836 (+2 ms)Running coprocessor post-close hooks at 1732686323844 (+8 ms)Closed at 1732686323845 (+1 ms) 2024-11-27T05:45:23,845 DEBUG [RS_CLOSE_META-regionserver/4da2d47fc9ac:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T05:45:23,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4da2d47fc9ac,45831,1732686322949 2024-11-27T05:45:23,865 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T05:45:23,865 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T05:45:23,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T05:45:23,898 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T05:45:23,898 DEBUG [M:0;4da2d47fc9ac:45173 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-27T05:45:23,898 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:23,898 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T05:45:23,898 DEBUG [M:0;4da2d47fc9ac:45173 {}] master.ActiveMasterManager(353): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-27T05:45:23,898 INFO [M:0;4da2d47fc9ac:45173 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/.lastflushedseqids 2024-11-27T05:45:23,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T05:45:23,904 INFO [M:0;4da2d47fc9ac:45173 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T05:45:23,904 INFO [M:0;4da2d47fc9ac:45173 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=true 2024-11-27T05:45:23,905 DEBUG [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T05:45:23,905 INFO [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:23,905 DEBUG [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:23,905 DEBUG [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T05:45:23,905 DEBUG [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:23,906 ERROR [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1960): Memstore data size is 33921 in region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:23,906 INFO [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T05:45:23,906 DEBUG [M:0;4da2d47fc9ac:45173 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732686323905Disabling compacts and flushes for region at 1732686323905Disabling writes for close at 1732686323905Writing region close event to WAL at 1732686323906 (+1 ms)Closed at 1732686323906 2024-11-27T05:45:23,906 INFO [M:0;4da2d47fc9ac:45173 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T05:45:23,906 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T05:45:23,906 INFO [M:0;4da2d47fc9ac:45173 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45173 2024-11-27T05:45:23,906 INFO [M:0;4da2d47fc9ac:45173 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T05:45:23,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:23,988 INFO [RS:2;4da2d47fc9ac:45831 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T05:45:23,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45831-0x1017ad309e40003, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:23,988 INFO [RS:2;4da2d47fc9ac:45831 {}] regionserver.HRegionServer(1031): Exiting; stopping=4da2d47fc9ac,45831,1732686322949; zookeeper connection closed. 2024-11-27T05:45:23,988 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@40dedfa6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@40dedfa6 2024-11-27T05:45:24,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:24,019 INFO [M:0;4da2d47fc9ac:45173 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T05:45:24,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45173-0x1017ad309e40000, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:24,034 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(976): stopping server 4da2d47fc9ac,37641,1732686319363; all regions closed. 2024-11-27T05:45:24,038 DEBUG [RS:0;4da2d47fc9ac:37641 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:24,038 INFO [RS:0;4da2d47fc9ac:37641 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C37641%2C1732686319363.meta:.meta(num 1732686323493) 2024-11-27T05:45:24,041 DEBUG [RS:0;4da2d47fc9ac:37641 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/8481426d-5123-44d1-acd5-c9e69033f666/6fe9dea0-b628-aee1-d4d8-1a83b4477b93/oldWALs 2024-11-27T05:45:24,041 INFO [RS:0;4da2d47fc9ac:37641 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4da2d47fc9ac%2C37641%2C1732686319363:(num 1732686320257) 2024-11-27T05:45:24,041 DEBUG [RS:0;4da2d47fc9ac:37641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T05:45:24,041 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T05:45:24,041 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T05:45:24,042 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.ChoreService(370): Chore service for: regionserver/4da2d47fc9ac:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T05:45:24,042 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T05:45:24,042 INFO [regionserver/4da2d47fc9ac:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T05:45:24,042 INFO [RS:0;4da2d47fc9ac:37641 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37641 2024-11-27T05:45:24,061 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4da2d47fc9ac,37641,1732686319363 2024-11-27T05:45:24,061 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T05:45:24,214 INFO [RS:0;4da2d47fc9ac:37641 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T05:45:24,214 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:24,215 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37641-0x1017ad309e40001, quorum=127.0.0.1:56667, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T05:45:24,215 INFO [RS:0;4da2d47fc9ac:37641 {}] regionserver.HRegionServer(1031): Exiting; stopping=4da2d47fc9ac,37641,1732686319363; zookeeper connection closed. 2024-11-27T05:45:24,216 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e8db77f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e8db77f 2024-11-27T05:45:24,217 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-27T05:45:24,223 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T05:45:24,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-27T05:45:24,225 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T05:45:24,231 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 51) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=345 (was 340) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7992 (was 8064)