2024-12-09 01:45:20,310 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-09 01:45:20,331 main DEBUG Took 0.019059 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 01:45:20,331 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 01:45:20,331 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 01:45:20,332 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 01:45:20,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,340 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 01:45:20,356 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,357 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,358 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,359 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,360 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,361 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,362 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,362 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,363 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,364 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,364 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,365 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,365 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,366 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:45:20,366 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,367 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 01:45:20,368 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:45:20,369 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 01:45:20,371 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 01:45:20,371 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 01:45:20,373 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 01:45:20,373 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 01:45:20,381 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 01:45:20,384 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 01:45:20,385 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 01:45:20,385 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 01:45:20,386 main DEBUG createAppenders(={Console}) 2024-12-09 01:45:20,386 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-09 01:45:20,387 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-09 01:45:20,387 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-09 01:45:20,387 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 01:45:20,388 main DEBUG OutputStream closed 2024-12-09 01:45:20,388 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 01:45:20,388 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 01:45:20,388 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-09 01:45:20,457 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 01:45:20,459 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 01:45:20,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 01:45:20,461 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 01:45:20,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 01:45:20,462 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 01:45:20,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 01:45:20,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 01:45:20,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 01:45:20,463 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 01:45:20,464 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 01:45:20,464 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 01:45:20,464 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 01:45:20,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 01:45:20,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 01:45:20,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 01:45:20,465 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 01:45:20,466 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 01:45:20,468 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 01:45:20,469 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-09 01:45:20,469 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 01:45:20,470 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-09T01:45:20,483 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-09 01:45:20,486 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 01:45:20,487 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T01:45:20,725 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0 2024-12-09T01:45:20,752 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=12376 2024-12-09T01:45:20,878 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T01:45:21,001 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:21,018 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/cluster_0a37ac4b-822c-03b1-9360-9fa5ec9dc69e, deleteOnExit=true 2024-12-09T01:45:21,127 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/cluster_0a37ac4b-822c-03b1-9360-9fa5ec9dc69e/zookeeper_0, clientPort=49324, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/cluster_0a37ac4b-822c-03b1-9360-9fa5ec9dc69e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/cluster_0a37ac4b-822c-03b1-9360-9fa5ec9dc69e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T01:45:21,140 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49324 2024-12-09T01:45:21,141 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:21,241 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba with version=8 2024-12-09T01:45:21,242 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/hbase-staging 2024-12-09T01:45:21,346 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T01:45:21,642 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:45:21,653 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:21,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:21,662 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:45:21,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:21,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:45:21,806 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T01:45:21,911 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:45:21,960 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T01:45:21,974 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T01:45:21,978 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:45:22,026 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14359 (auto-detected) 2024-12-09T01:45:22,028 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T01:45:22,052 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45355 2024-12-09T01:45:22,077 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45355 connecting to ZooKeeper ensemble=127.0.0.1:49324 2024-12-09T01:45:22,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:453550x0, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:45:22,116 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45355-0x100747364b80000 connected 2024-12-09T01:45:22,144 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:22,145 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:22,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:22,157 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba, hbase.cluster.distributed=false 2024-12-09T01:45:22,184 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:45:22,193 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45355 2024-12-09T01:45:22,194 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45355 2024-12-09T01:45:22,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45355 2024-12-09T01:45:22,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45355 2024-12-09T01:45:22,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45355 2024-12-09T01:45:22,292 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:45:22,294 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:22,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:22,295 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:45:22,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:22,295 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:45:22,298 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T01:45:22,302 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:45:22,303 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40111 2024-12-09T01:45:22,305 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40111 connecting to ZooKeeper ensemble=127.0.0.1:49324 2024-12-09T01:45:22,306 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:22,306 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:22,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401110x0, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:45:22,311 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40111-0x100747364b80001 connected 2024-12-09T01:45:22,311 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:22,317 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T01:45:22,330 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.cache 2024-12-09T01:45:22,338 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T01:45:22,340 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T01:45:22,342 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T01:45:22,346 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T01:45:22,346 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T01:45:22,352 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T01:45:22,357 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T01:45:22,357 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.persistence is complete. 2024-12-09T01:45:22,360 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T01:45:22,363 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T01:45:22,369 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:45:22,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40111 2024-12-09T01:45:22,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40111 2024-12-09T01:45:22,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40111 2024-12-09T01:45:22,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40111 2024-12-09T01:45:22,376 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40111 2024-12-09T01:45:22,395 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:45:22,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:22,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:22,396 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:45:22,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:22,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:45:22,397 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T01:45:22,397 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:45:22,400 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36001 2024-12-09T01:45:22,402 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36001 connecting to ZooKeeper ensemble=127.0.0.1:49324 2024-12-09T01:45:22,402 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:22,403 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:22,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360010x0, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:45:22,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360010x0, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:22,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36001-0x100747364b80002 connected 2024-12-09T01:45:22,413 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T01:45:22,417 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.cache 2024-12-09T01:45:22,417 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T01:45:22,417 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T01:45:22,417 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T01:45:22,418 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T01:45:22,418 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T01:45:22,419 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T01:45:22,419 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.persistence is complete. 2024-12-09T01:45:22,420 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T01:45:22,424 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T01:45:22,426 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T01:45:22,428 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:45:22,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36001 2024-12-09T01:45:22,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36001 2024-12-09T01:45:22,432 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36001 2024-12-09T01:45:22,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36001 2024-12-09T01:45:22,437 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36001 2024-12-09T01:45:22,463 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef6f18c58dc9:45355 2024-12-09T01:45:22,465 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:22,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:22,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:22,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:22,477 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:22,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T01:45:22,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T01:45:22,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:22,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:22,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:22,507 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T01:45:22,509 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef6f18c58dc9,45355,1733708721433 from backup master directory 2024-12-09T01:45:22,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:22,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:22,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:22,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:22,514 WARN [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:45:22,514 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:22,516 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T01:45:22,519 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T01:45:22,529 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/hbase.id] with ID: 31b707b1-053b-4716-a818-bec68bbe5c41 2024-12-09T01:45:22,529 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.tmp/hbase.id 2024-12-09T01:45:22,536 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/hbase.id] 2024-12-09T01:45:22,583 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:22,589 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T01:45:22,619 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-12-09T01:45:22,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:22,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:22,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:22,653 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:45:22,657 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T01:45:22,711 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:45:22,717 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:22,789 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store 2024-12-09T01:45:22,820 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T01:45:22,824 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:22,826 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:45:22,827 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:22,827 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:22,829 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:45:22,829 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:22,830 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:22,830 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733708722826Disabling compacts and flushes for region at 1733708722826Disabling writes for close at 1733708722829 (+3 ms)Writing region close event to WAL at 1733708722829Closed at 1733708722829 2024-12-09T01:45:22,831 WARN [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/.initializing 2024-12-09T01:45:22,831 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/WALs/ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:22,845 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:45:22,859 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef6f18c58dc9%2C45355%2C1733708721433, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/WALs/ef6f18c58dc9,45355,1733708721433, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/oldWALs, maxLogs=10 2024-12-09T01:45:22,901 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/WALs/ef6f18c58dc9,45355,1733708721433/ef6f18c58dc9%2C45355%2C1733708721433.1733708722863 2024-12-09T01:45:22,902 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:22,902 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:22,903 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:22,907 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:22,908 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:22,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,000 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T01:45:23,006 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,008 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:23,009 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,016 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T01:45:23,017 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,017 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:23,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,023 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T01:45:23,023 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,024 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:23,024 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T01:45:23,030 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:23,031 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,036 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,037 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,042 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,043 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,047 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:45:23,050 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:23,060 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:45:23,061 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67704707, jitterRate=0.008878752589225769}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:45:23,071 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733708722921Initializing all the Stores at 1733708722922 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708722922Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708722923 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708722924 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708722924Cleaning up temporary data from old regions at 1733708723043 (+119 ms)Region opened successfully at 1733708723071 (+28 ms) 2024-12-09T01:45:23,073 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T01:45:23,115 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346c3706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:45:23,161 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T01:45:23,174 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T01:45:23,174 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T01:45:23,178 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T01:45:23,178 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T01:45:23,184 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-09T01:45:23,184 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T01:45:23,212 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T01:45:23,221 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T01:45:23,224 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T01:45:23,228 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T01:45:23,229 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T01:45:23,231 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T01:45:23,234 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T01:45:23,237 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T01:45:23,239 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T01:45:23,241 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T01:45:23,243 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T01:45:23,263 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T01:45:23,265 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T01:45:23,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:23,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:23,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:23,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,282 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef6f18c58dc9,45355,1733708721433, sessionid=0x100747364b80000, setting cluster-up flag (Was=false) 2024-12-09T01:45:23,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,303 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T01:45:23,306 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:23,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:23,318 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T01:45:23,320 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:23,327 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T01:45:23,343 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(746): ClusterId : 31b707b1-053b-4716-a818-bec68bbe5c41 2024-12-09T01:45:23,343 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(746): ClusterId : 31b707b1-053b-4716-a818-bec68bbe5c41 2024-12-09T01:45:23,346 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T01:45:23,346 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T01:45:23,352 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T01:45:23,352 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T01:45:23,352 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T01:45:23,352 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T01:45:23,355 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T01:45:23,356 DEBUG [RS:1;ef6f18c58dc9:36001 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5beefb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:45:23,357 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T01:45:23,358 DEBUG [RS:0;ef6f18c58dc9:40111 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f11a7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:45:23,378 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ef6f18c58dc9:36001 2024-12-09T01:45:23,378 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef6f18c58dc9:40111 2024-12-09T01:45:23,382 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T01:45:23,382 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T01:45:23,382 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T01:45:23,382 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T01:45:23,382 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T01:45:23,382 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T01:45:23,386 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,45355,1733708721433 with port=36001, startcode=1733708722395 2024-12-09T01:45:23,386 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,45355,1733708721433 with port=40111, startcode=1733708722252 2024-12-09T01:45:23,402 DEBUG [RS:0;ef6f18c58dc9:40111 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T01:45:23,402 DEBUG [RS:1;ef6f18c58dc9:36001 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T01:45:23,414 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:23,428 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T01:45:23,437 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T01:45:23,444 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef6f18c58dc9,45355,1733708721433 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T01:45:23,459 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:23,460 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:23,460 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:23,460 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:23,461 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef6f18c58dc9:0, corePoolSize=10, maxPoolSize=10 2024-12-09T01:45:23,461 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,461 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:45:23,462 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,473 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:23,474 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T01:45:23,478 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733708753478 2024-12-09T01:45:23,482 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,483 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T01:45:23,483 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T01:45:23,484 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T01:45:23,489 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41161, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T01:45:23,489 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55237, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T01:45:23,489 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T01:45:23,490 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T01:45:23,490 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T01:45:23,490 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T01:45:23,494 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,498 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T01:45:23,498 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45355 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:45:23,499 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T01:45:23,500 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T01:45:23,504 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T01:45:23,505 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T01:45:23,509 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45355 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:45:23,515 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708723508,5,FailOnTimeoutGroup] 2024-12-09T01:45:23,516 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708723515,5,FailOnTimeoutGroup] 2024-12-09T01:45:23,516 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T01:45:23,516 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,517 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T01:45:23,517 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:23,518 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,519 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,541 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T01:45:23,541 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-09T01:45:23,541 WARN [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T01:45:23,541 WARN [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-09T01:45:23,542 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:23,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T01:45:23,553 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:45:23,553 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:23,554 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T01:45:23,558 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:45:23,558 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:23,559 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T01:45:23,563 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:45:23,563 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:23,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T01:45:23,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:45:23,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:23,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:23,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:45:23,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:23,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:23,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:45:23,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:45:23,572 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:45:23,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:45:23,583 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:45:23,584 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59660066, jitterRate=-0.11099573969841003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:45:23,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733708723542Initializing all the Stores at 1733708723542Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708723542Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708723544 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708723544Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708723544Cleaning up temporary data from old regions at 1733708723572 (+28 ms)Region opened successfully at 1733708723588 (+16 ms) 2024-12-09T01:45:23,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:45:23,589 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:45:23,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:45:23,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:45:23,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:45:23,591 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:45:23,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733708723588Disabling compacts and flushes for region at 1733708723588Disabling writes for close at 1733708723589 (+1 ms)Writing region close event to WAL at 1733708723590 (+1 ms)Closed at 1733708723591 (+1 ms) 2024-12-09T01:45:23,594 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:23,594 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T01:45:23,604 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T01:45:23,615 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:45:23,617 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T01:45:23,643 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,45355,1733708721433 with port=36001, startcode=1733708722395 2024-12-09T01:45:23,643 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,45355,1733708721433 with port=40111, startcode=1733708722252 2024-12-09T01:45:23,645 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45355 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:23,648 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45355 {}] master.ServerManager(517): Registering regionserver=ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:23,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45355 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:23,655 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45355 {}] master.ServerManager(517): Registering regionserver=ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:23,656 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:23,656 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T01:45:23,656 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T01:45:23,658 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:23,658 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T01:45:23,658 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T01:45:23,664 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:23,665 DEBUG [RS:0;ef6f18c58dc9:40111 {}] zookeeper.ZKUtil(111): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:23,665 WARN [RS:0;ef6f18c58dc9:40111 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:45:23,665 DEBUG [RS:1;ef6f18c58dc9:36001 {}] zookeeper.ZKUtil(111): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:23,665 INFO [RS:0;ef6f18c58dc9:40111 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:23,665 WARN [RS:1;ef6f18c58dc9:36001 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:45:23,665 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:23,665 INFO [RS:1;ef6f18c58dc9:36001 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:23,666 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:23,668 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef6f18c58dc9,36001,1733708722395] 2024-12-09T01:45:23,668 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef6f18c58dc9,40111,1733708722252] 2024-12-09T01:45:23,693 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T01:45:23,693 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T01:45:23,712 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T01:45:23,712 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T01:45:23,718 INFO [RS:1;ef6f18c58dc9:36001 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T01:45:23,718 INFO [RS:0;ef6f18c58dc9:40111 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T01:45:23,718 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,718 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,720 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T01:45:23,720 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T01:45:23,725 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T01:45:23,725 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T01:45:23,727 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,727 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,728 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,728 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,728 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,728 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,728 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,728 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,728 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,728 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,729 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,729 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,729 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:45:23,729 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:45:23,729 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,729 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,729 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,729 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,730 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:23,731 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:23,731 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:23,731 DEBUG [RS:0;ef6f18c58dc9:40111 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:23,731 DEBUG [RS:1;ef6f18c58dc9:36001 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:23,740 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,740 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,740 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,740 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,740 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,740 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,741 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,741 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,741 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,741 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,741 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,40111,1733708722252-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:45:23,741 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,36001,1733708722395-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:45:23,763 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T01:45:23,763 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T01:45:23,765 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,36001,1733708722395-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,766 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,40111,1733708722252-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,766 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,766 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,766 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.Replication(171): ef6f18c58dc9,36001,1733708722395 started 2024-12-09T01:45:23,766 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.Replication(171): ef6f18c58dc9,40111,1733708722252 started 2024-12-09T01:45:23,768 WARN [ef6f18c58dc9:45355 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:45:23,786 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,786 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1482): Serving as ef6f18c58dc9,36001,1733708722395, RpcServer on ef6f18c58dc9/172.17.0.2:36001, sessionid=0x100747364b80002 2024-12-09T01:45:23,787 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T01:45:23,787 DEBUG [RS:1;ef6f18c58dc9:36001 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:23,787 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,36001,1733708722395' 2024-12-09T01:45:23,787 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T01:45:23,789 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T01:45:23,789 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T01:45:23,789 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T01:45:23,790 DEBUG [RS:1;ef6f18c58dc9:36001 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:23,790 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,36001,1733708722395' 2024-12-09T01:45:23,790 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T01:45:23,790 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T01:45:23,791 DEBUG [RS:1;ef6f18c58dc9:36001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T01:45:23,791 INFO [RS:1;ef6f18c58dc9:36001 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T01:45:23,791 INFO [RS:1;ef6f18c58dc9:36001 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T01:45:23,793 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:23,793 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1482): Serving as ef6f18c58dc9,40111,1733708722252, RpcServer on ef6f18c58dc9/172.17.0.2:40111, sessionid=0x100747364b80001 2024-12-09T01:45:23,793 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T01:45:23,793 DEBUG [RS:0;ef6f18c58dc9:40111 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:23,793 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,40111,1733708722252' 2024-12-09T01:45:23,794 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T01:45:23,794 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T01:45:23,795 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T01:45:23,795 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T01:45:23,795 DEBUG [RS:0;ef6f18c58dc9:40111 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:23,795 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,40111,1733708722252' 2024-12-09T01:45:23,795 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T01:45:23,796 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T01:45:23,797 DEBUG [RS:0;ef6f18c58dc9:40111 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T01:45:23,797 INFO [RS:0;ef6f18c58dc9:40111 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T01:45:23,797 INFO [RS:0;ef6f18c58dc9:40111 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T01:45:23,897 INFO [RS:1;ef6f18c58dc9:36001 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:45:23,898 INFO [RS:1;ef6f18c58dc9:36001 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef6f18c58dc9%2C36001%2C1733708722395, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,36001,1733708722395, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs, maxLogs=55 2024-12-09T01:45:23,898 INFO [RS:0;ef6f18c58dc9:40111 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:45:23,899 INFO [RS:0;ef6f18c58dc9:40111 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef6f18c58dc9%2C40111%2C1733708722252, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,40111,1733708722252, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs, maxLogs=55 2024-12-09T01:45:23,920 INFO [RS:0;ef6f18c58dc9:40111 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,40111,1733708722252/ef6f18c58dc9%2C40111%2C1733708722252.1733708723901 2024-12-09T01:45:23,920 INFO [RS:1;ef6f18c58dc9:36001 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,36001,1733708722395/ef6f18c58dc9%2C36001%2C1733708722395.1733708723900 2024-12-09T01:45:23,921 DEBUG [RS:0;ef6f18c58dc9:40111 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:23,921 DEBUG [RS:1;ef6f18c58dc9:36001 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:24,021 DEBUG [ef6f18c58dc9:45355 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-09T01:45:24,032 DEBUG [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(204): Hosts are {ef6f18c58dc9=0} racks are {/default-rack=0} 2024-12-09T01:45:24,036 DEBUG [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T01:45:24,036 DEBUG [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T01:45:24,036 DEBUG [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T01:45:24,036 DEBUG [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T01:45:24,036 INFO [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T01:45:24,036 INFO [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T01:45:24,036 DEBUG [ef6f18c58dc9:45355 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T01:45:24,045 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:24,051 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef6f18c58dc9,40111,1733708722252, state=OPENING 2024-12-09T01:45:24,058 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T01:45:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:24,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:24,060 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:24,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:24,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:24,062 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:45:24,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,40111,1733708722252}] 2024-12-09T01:45:24,240 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T01:45:24,243 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32851, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T01:45:24,253 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T01:45:24,253 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:24,254 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T01:45:24,256 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef6f18c58dc9%2C40111%2C1733708722252.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,40111,1733708722252, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs, maxLogs=55 2024-12-09T01:45:24,279 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,40111,1733708722252/ef6f18c58dc9%2C40111%2C1733708722252.meta.1733708724258.meta 2024-12-09T01:45:24,279 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:24,280 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:24,282 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T01:45:24,285 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T01:45:24,289 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T01:45:24,294 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T01:45:24,294 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:24,295 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T01:45:24,295 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T01:45:24,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T01:45:24,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:45:24,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:24,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:24,298 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T01:45:24,299 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:45:24,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:24,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:24,300 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T01:45:24,301 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:45:24,301 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:24,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:24,301 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T01:45:24,302 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:45:24,302 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:24,302 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:24,302 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:45:24,303 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:24,304 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:24,305 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:45:24,305 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:45:24,305 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:45:24,306 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:45:24,307 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74153167, jitterRate=0.10496829450130463}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:45:24,307 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T01:45:24,309 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733708724295Writing region info on filesystem at 1733708724295Initializing all the Stores at 1733708724296 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708724296Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708724296Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708724296Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708724296Cleaning up temporary data from old regions at 1733708724305 (+9 ms)Running coprocessor post-open hooks at 1733708724307 (+2 ms)Region opened successfully at 1733708724309 (+2 ms) 2024-12-09T01:45:24,315 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733708724230 2024-12-09T01:45:24,326 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T01:45:24,327 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T01:45:24,327 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:24,329 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef6f18c58dc9,40111,1733708722252, state=OPEN 2024-12-09T01:45:24,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:45:24,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:45:24,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:45:24,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:24,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:24,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:24,332 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:24,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T01:45:24,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,40111,1733708722252 in 268 msec 2024-12-09T01:45:24,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T01:45:24,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 734 msec 2024-12-09T01:45:24,341 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:24,341 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T01:45:24,363 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:45:24,364 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,40111,1733708722252, seqNum=-1] 2024-12-09T01:45:24,385 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:45:24,389 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59013, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:45:24,405 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0450 sec 2024-12-09T01:45:24,405 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733708724405, completionTime=-1 2024-12-09T01:45:24,409 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-09T01:45:24,409 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T01:45:24,439 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T01:45:24,439 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733708784439 2024-12-09T01:45:24,439 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733708844439 2024-12-09T01:45:24,439 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 30 msec 2024-12-09T01:45:24,441 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-09T01:45:24,448 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,45355,1733708721433-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:24,448 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,45355,1733708721433-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:24,448 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,45355,1733708721433-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:24,450 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef6f18c58dc9:45355, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:24,451 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:24,451 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:24,456 DEBUG [master/ef6f18c58dc9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T01:45:24,483 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.969sec 2024-12-09T01:45:24,485 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T01:45:24,487 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T01:45:24,488 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T01:45:24,489 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T01:45:24,489 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T01:45:24,490 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,45355,1733708721433-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:45:24,490 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,45355,1733708721433-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T01:45:24,492 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T01:45:24,493 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T01:45:24,493 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,45355,1733708721433-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:24,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:45:24,557 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T01:45:24,557 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T01:45:24,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,45355,-1 for getting cluster id 2024-12-09T01:45:24,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:45:24,572 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '31b707b1-053b-4716-a818-bec68bbe5c41' 2024-12-09T01:45:24,575 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:45:24,575 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "31b707b1-053b-4716-a818-bec68bbe5c41" 2024-12-09T01:45:24,575 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784105d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:45:24,576 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,45355,-1] 2024-12-09T01:45:24,578 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:45:24,580 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:24,581 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:45:24,584 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:45:24,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:45:24,591 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,40111,1733708722252, seqNum=-1] 2024-12-09T01:45:24,592 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:45:24,595 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:45:24,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:24,636 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:45:24,641 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:24,647 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5b25c33c 2024-12-09T01:45:24,648 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:45:24,651 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:45:24,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733708724632', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:45:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 2024-12-09T01:45:24,666 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:45:24,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733708724632" procId is: 4 2024-12-09T01:45:24,669 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:24,671 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:45:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:24,687 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 957e404cda065da371c5ed55df4442a7, NAME => 'testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733708724632', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:24,695 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:24,695 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(1722): Closing 957e404cda065da371c5ed55df4442a7, disabling compactions & flushes 2024-12-09T01:45:24,695 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:24,695 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:24,695 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. after waiting 0 ms 2024-12-09T01:45:24,695 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:24,695 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:24,695 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733708724632-pool-0 {}] regionserver.HRegion(1676): Region close journal for 957e404cda065da371c5ed55df4442a7: Waiting for close lock at 1733708724695Disabling compacts and flushes for region at 1733708724695Disabling writes for close at 1733708724695Writing region close event to WAL at 1733708724695Closed at 1733708724695 2024-12-09T01:45:24,697 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:45:24,704 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733708724697"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708724697"}]},"ts":"1733708724697"} 2024-12-09T01:45:24,709 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:45:24,711 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:45:24,714 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733708724632","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708724711"}]},"ts":"1733708724711"} 2024-12-09T01:45:24,719 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733708724632, state=ENABLING in hbase:meta 2024-12-09T01:45:24,719 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ef6f18c58dc9=0} racks are {/default-rack=0} 2024-12-09T01:45:24,720 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T01:45:24,720 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T01:45:24,720 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T01:45:24,720 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T01:45:24,720 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T01:45:24,720 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T01:45:24,720 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T01:45:24,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, ASSIGN}] 2024-12-09T01:45:24,724 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, ASSIGN 2024-12-09T01:45:24,726 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,40111,1733708722252; forceNewPlan=false, retain=false 2024-12-09T01:45:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:24,878 INFO [ef6f18c58dc9:45355 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T01:45:24,879 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=957e404cda065da371c5ed55df4442a7, regionState=OPENING, regionLocation=ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:24,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, ASSIGN because future has completed 2024-12-09T01:45:24,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,40111,1733708722252}] 2024-12-09T01:45:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:25,040 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:25,040 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 957e404cda065da371c5ed55df4442a7, NAME => 'testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:25,040 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733708724632 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,041 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:25,041 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,041 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,042 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,047 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 957e404cda065da371c5ed55df4442a7 columnFamilyName family 2024-12-09T01:45:25,047 DEBUG [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:25,047 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.HStore(327): Store=957e404cda065da371c5ed55df4442a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:25,048 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,049 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,049 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,049 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,049 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,050 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,056 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:45:25,057 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 957e404cda065da371c5ed55df4442a7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61566012, jitterRate=-0.08259493112564087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:45:25,057 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:25,058 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 957e404cda065da371c5ed55df4442a7: Running coprocessor pre-open hook at 1733708725041Writing region info on filesystem at 1733708725041Initializing all the Stores at 1733708725041Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708725041Cleaning up temporary data from old regions at 1733708725049 (+8 ms)Running coprocessor post-open hooks at 1733708725057 (+8 ms)Region opened successfully at 1733708725057 2024-12-09T01:45:25,059 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7., pid=6, masterSystemTime=1733708725036 2024-12-09T01:45:25,062 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:25,062 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:25,063 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=957e404cda065da371c5ed55df4442a7, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:25,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,40111,1733708722252 because future has completed 2024-12-09T01:45:25,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T01:45:25,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,40111,1733708722252 in 183 msec 2024-12-09T01:45:25,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T01:45:25,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, ASSIGN in 347 msec 2024-12-09T01:45:25,072 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:45:25,072 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733708724632","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708725072"}]},"ts":"1733708725072"} 2024-12-09T01:45:25,074 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733708724632, state=ENABLED in hbase:meta 2024-12-09T01:45:25,075 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:45:25,077 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733708724632 in 415 msec 2024-12-09T01:45:25,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:25,310 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733708724632 get assigned. Timeout = 60000ms 2024-12-09T01:45:25,310 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733708724632 completed 2024-12-09T01:45:25,313 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:45:25,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733708724632 assigned to meta. Checking AM states. 2024-12-09T01:45:25,319 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:45:25,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733708724632 assigned. 2024-12-09T01:45:25,328 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733708724632', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7., hostname=ef6f18c58dc9,40111,1733708722252, seqNum=2] 2024-12-09T01:45:25,341 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 957e404cda065da371c5ed55df4442a7 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-09T01:45:25,373 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f24603f79002472b8bf66d4047cf0b82 2024-12-09T01:45:25,411 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/.tmp/family/f24603f79002472b8bf66d4047cf0b82 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-09T01:45:25,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/.tmp/family/f24603f79002472b8bf66d4047cf0b82 2024-12-09T01:45:25,453 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/.tmp/family/f24603f79002472b8bf66d4047cf0b82 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82 2024-12-09T01:45:25,455 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82, delay=916 ms 2024-12-09T01:45:25,458 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82, entries=2, sequenceid=6, filesize=4.9 K 2024-12-09T01:45:25,466 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 957e404cda065da371c5ed55df4442a7 in 124ms, sequenceid=6, compaction requested=false 2024-12-09T01:45:25,467 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733708724632' 2024-12-09T01:45:25,470 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 957e404cda065da371c5ed55df4442a7: 2024-12-09T01:45:26,374 DEBUG [hfile-prefetch-1733708725455 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f24603f79002472b8bf66d4047cf0b82_0. 2024-12-09T01:45:26,378 DEBUG [hfile-prefetch-1733708725455 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f24603f79002472b8bf66d4047cf0b82_117. 2024-12-09T01:45:26,379 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82, with total blocks 2, and data blocks 1 2024-12-09T01:45:26,379 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-09T01:45:26,380 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-09T01:45:26,380 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82. Will sleep 100ms and try the verification again. 2024-12-09T01:45:26,427 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T01:45:26,427 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T01:45:26,428 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:26,428 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:26,444 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:26,444 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708726427 2024-12-09T01:45:26,444 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T01:45:26,480 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82, with total blocks 2, and data blocks 1 2024-12-09T01:45:26,481 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-09T01:45:26,481 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2293): found block for file f24603f79002472b8bf66d4047cf0b82 in the backing map. Acquiring read lock for offset 0 2024-12-09T01:45:26,481 DEBUG [hfile-prefetch-1733708725455 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82 has now been fully cached. 2024-12-09T01:45:26,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=957e404cda065da371c5ed55df4442a7, source=ef6f18c58dc9,40111,1733708722252, destination=ef6f18c58dc9,36001,1733708722395, warming up region on ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:26,481 DEBUG [hfile-prefetch-1733708725455 {}] hfile.PrefetchExecutor(121): Prefetch completed for f24603f79002472b8bf66d4047cf0b82 2024-12-09T01:45:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T01:45:26,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=957e404cda065da371c5ed55df4442a7, source=ef6f18c58dc9,40111,1733708722252, destination=ef6f18c58dc9,36001,1733708722395, running balancer 2024-12-09T01:45:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, REOPEN/MOVE 2024-12-09T01:45:26,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33735, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T01:45:26,485 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, REOPEN/MOVE 2024-12-09T01:45:26,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(7855): Warmup {ENCODED => 957e404cda065da371c5ed55df4442a7, NAME => 'testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:26,487 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=957e404cda065da371c5ed55df4442a7, regionState=CLOSING, regionLocation=ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:26,488 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,488 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 957e404cda065da371c5ed55df4442a7 columnFamilyName family 2024-12-09T01:45:26,489 DEBUG [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:26,490 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, REOPEN/MOVE because future has completed 2024-12-09T01:45:26,491 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-09T01:45:26,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,40111,1733708722252}] 2024-12-09T01:45:26,492 DEBUG [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82 2024-12-09T01:45:26,492 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.HStore(327): Store=957e404cda065da371c5ed55df4442a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(1722): Closing 957e404cda065da371c5ed55df4442a7, disabling compactions & flushes 2024-12-09T01:45:26,492 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. after waiting 1 ms 2024-12-09T01:45:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,494 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36001 {}] regionserver.HRegion(1676): Region close journal for 957e404cda065da371c5ed55df4442a7: Waiting for close lock at 1733708726492Disabling compacts and flushes for region at 1733708726492Disabling writes for close at 1733708726493 (+1 ms)Writing region close event to WAL at 1733708726494 (+1 ms)Closed at 1733708726494 2024-12-09T01:45:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45355 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-09T01:45:26,649 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,649 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-09T01:45:26,650 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 957e404cda065da371c5ed55df4442a7, disabling compactions & flushes 2024-12-09T01:45:26,650 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,650 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,650 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. after waiting 0 ms 2024-12-09T01:45:26,650 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,652 DEBUG [StoreFileCloser-957e404cda065da371c5ed55df4442a7-family-1 {}] bucket.BucketCache(768): Removing file f24603f79002472b8bf66d4047cf0b82 for region 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,652 DEBUG [StoreFileCloser-957e404cda065da371c5ed55df4442a7-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file f24603f79002472b8bf66d4047cf0b82, starting offset: 0, end offset: 9223372036854775807 2024-12-09T01:45:26,654 DEBUG [StoreFileCloser-957e404cda065da371c5ed55df4442a7-family-1 {}] bucket.BucketCache(856): removed key f24603f79002472b8bf66d4047cf0b82_0 from back map with offset lock 2094080 in the evict process 2024-12-09T01:45:26,659 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-09T01:45:26,660 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,660 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 957e404cda065da371c5ed55df4442a7: Waiting for close lock at 1733708726650Running coprocessor pre-close hooks at 1733708726650Disabling compacts and flushes for region at 1733708726650Disabling writes for close at 1733708726650Writing region close event to WAL at 1733708726654 (+4 ms)Running coprocessor post-close hooks at 1733708726659 (+5 ms)Closed at 1733708726660 (+1 ms) 2024-12-09T01:45:26,661 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 957e404cda065da371c5ed55df4442a7 move to ef6f18c58dc9,36001,1733708722395 record at close sequenceid=6 2024-12-09T01:45:26,664 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,665 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=957e404cda065da371c5ed55df4442a7, regionState=CLOSED 2024-12-09T01:45:26,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,40111,1733708722252 because future has completed 2024-12-09T01:45:26,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T01:45:26,670 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,40111,1733708722252 in 178 msec 2024-12-09T01:45:26,671 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, REOPEN/MOVE; state=CLOSED, location=ef6f18c58dc9,36001,1733708722395; forceNewPlan=false, retain=false 2024-12-09T01:45:26,744 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T01:45:26,744 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T01:45:26,744 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:26,745 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:26,747 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T01:45:26,747 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T01:45:26,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:26,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:26,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:26,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708726744 2024-12-09T01:45:26,748 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T01:45:26,752 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:26,752 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708726747 2024-12-09T01:45:26,752 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T01:45:26,822 INFO [ef6f18c58dc9:45355 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T01:45:26,822 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=957e404cda065da371c5ed55df4442a7, regionState=OPENING, regionLocation=ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:26,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, REOPEN/MOVE because future has completed 2024-12-09T01:45:26,830 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,36001,1733708722395}] 2024-12-09T01:45:26,985 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:26,986 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 957e404cda065da371c5ed55df4442a7, NAME => 'testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:26,986 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733708724632 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,986 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:26,986 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,986 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,987 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,988 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 957e404cda065da371c5ed55df4442a7 columnFamilyName family 2024-12-09T01:45:26,988 DEBUG [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:26,991 DEBUG [StoreFileOpener-957e404cda065da371c5ed55df4442a7-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82, delay=979 ms 2024-12-09T01:45:26,992 DEBUG [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82 2024-12-09T01:45:26,992 INFO [StoreOpener-957e404cda065da371c5ed55df4442a7-1 {}] regionserver.HStore(327): Store=957e404cda065da371c5ed55df4442a7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:26,993 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,993 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,994 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,994 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,994 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,998 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:26,999 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 957e404cda065da371c5ed55df4442a7; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68957546, jitterRate=0.027547508478164673}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:45:26,999 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:27,000 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 957e404cda065da371c5ed55df4442a7: Running coprocessor pre-open hook at 1733708726987Writing region info on filesystem at 1733708726987Initializing all the Stores at 1733708726987Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708726987Cleaning up temporary data from old regions at 1733708726994 (+7 ms)Running coprocessor post-open hooks at 1733708726999 (+5 ms)Region opened successfully at 1733708727000 (+1 ms) 2024-12-09T01:45:27,000 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7., pid=9, masterSystemTime=1733708726982 2024-12-09T01:45:27,004 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:27,004 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:27,004 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=957e404cda065da371c5ed55df4442a7, regionState=OPEN, openSeqNum=10, regionLocation=ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:27,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,36001,1733708722395 because future has completed 2024-12-09T01:45:27,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-09T01:45:27,010 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 957e404cda065da371c5ed55df4442a7, server=ef6f18c58dc9,36001,1733708722395 in 178 msec 2024-12-09T01:45:27,012 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733708724632, region=957e404cda065da371c5ed55df4442a7, REOPEN/MOVE in 527 msec 2024-12-09T01:45:27,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T01:45:27,037 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:45:27,038 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:27,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:27,043 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:27,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:45:27,043 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T01:45:27,043 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-12-09T01:45:27,044 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef6f18c58dc9,45355,1733708721433 2024-12-09T01:45:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,046 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:45:27,047 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:45:27,047 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:27,047 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:27,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:27,047 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:27,047 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(878): Closing user regions 2024-12-09T01:45:27,048 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef6f18c58dc9,40111,1733708722252' ***** 2024-12-09T01:45:27,048 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T01:45:27,048 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:27,048 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef6f18c58dc9,36001,1733708722395' ***** 2024-12-09T01:45:27,048 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T01:45:27,048 INFO [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T01:45:27,049 INFO [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.persistence 2024-12-09T01:45:27,049 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T01:45:27,049 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T01:45:27,049 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T01:45:27,049 INFO [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T01:45:27,049 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T01:45:27,051 INFO [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T01:45:27,051 INFO [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/bucket.persistence 2024-12-09T01:45:27,051 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T01:45:27,051 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T01:45:27,051 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T01:45:27,051 INFO [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T01:45:27,051 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T01:45:27,059 DEBUG [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1487): Thread RS:1;ef6f18c58dc9:36001 started persisting bucket cache to file 2024-12-09T01:45:27,059 DEBUG [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:27,059 DEBUG [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:27,061 DEBUG [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1487): Thread RS:0;ef6f18c58dc9:40111 started persisting bucket cache to file 2024-12-09T01:45:27,061 DEBUG [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:27,062 DEBUG [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:27,063 DEBUG [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:27,063 DEBUG [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708727059 2024-12-09T01:45:27,063 DEBUG [RS:1;ef6f18c58dc9:36001 {}] bucket.BucketCache(1509): Thread RS:1;ef6f18c58dc9:36001 finished persisting bucket cache to file, renaming 2024-12-09T01:45:27,063 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T01:45:27,064 INFO [RS:1;ef6f18c58dc9:36001 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T01:45:27,064 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T01:45:27,064 INFO [RS:1;ef6f18c58dc9:36001 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T01:45:27,064 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(3091): Received CLOSE for 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:27,064 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(959): stopping server ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:27,065 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:45:27,065 INFO [RS:1;ef6f18c58dc9:36001 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ef6f18c58dc9:36001. 2024-12-09T01:45:27,065 DEBUG [RS:1;ef6f18c58dc9:36001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:27,065 DEBUG [RS:1;ef6f18c58dc9:36001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:27,065 DEBUG [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:27,065 DEBUG [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708727061 2024-12-09T01:45:27,065 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T01:45:27,065 DEBUG [RS:0;ef6f18c58dc9:40111 {}] bucket.BucketCache(1509): Thread RS:0;ef6f18c58dc9:40111 finished persisting bucket cache to file, renaming 2024-12-09T01:45:27,065 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T01:45:27,065 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1325): Online Regions={957e404cda065da371c5ed55df4442a7=testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7.} 2024-12-09T01:45:27,065 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T01:45:27,065 INFO [RS:0;ef6f18c58dc9:40111 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T01:45:27,065 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T01:45:27,065 INFO [RS:0;ef6f18c58dc9:40111 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T01:45:27,065 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 957e404cda065da371c5ed55df4442a7, disabling compactions & flushes 2024-12-09T01:45:27,066 DEBUG [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1351): Waiting on 957e404cda065da371c5ed55df4442a7 2024-12-09T01:45:27,066 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(959): stopping server ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:27,066 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:27,066 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:45:27,066 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:27,066 INFO [RS:0;ef6f18c58dc9:40111 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef6f18c58dc9:40111. 2024-12-09T01:45:27,066 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. after waiting 0 ms 2024-12-09T01:45:27,066 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:27,066 DEBUG [RS:0;ef6f18c58dc9:40111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:27,066 DEBUG [RS:0;ef6f18c58dc9:40111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:27,066 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T01:45:27,066 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T01:45:27,066 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T01:45:27,066 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T01:45:27,067 DEBUG [StoreFileCloser-957e404cda065da371c5ed55df4442a7-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/family/f24603f79002472b8bf66d4047cf0b82 2024-12-09T01:45:27,067 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T01:45:27,067 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T01:45:27,067 DEBUG [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:45:27,067 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T01:45:27,067 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:45:27,067 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:45:27,067 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:45:27,067 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:45:27,067 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:45:27,068 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-09T01:45:27,072 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnRegionMove1733708724632/957e404cda065da371c5ed55df4442a7/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-09T01:45:27,072 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:27,072 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 957e404cda065da371c5ed55df4442a7: Waiting for close lock at 1733708727065Running coprocessor pre-close hooks at 1733708727065Disabling compacts and flushes for region at 1733708727065Disabling writes for close at 1733708727066 (+1 ms)Writing region close event to WAL at 1733708727067 (+1 ms)Running coprocessor post-close hooks at 1733708727072 (+5 ms)Closed at 1733708727072 2024-12-09T01:45:27,072 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7. 2024-12-09T01:45:27,078 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f6dea2a7d0a74e53a357c2d38f52cede 2024-12-09T01:45:27,097 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/info/f6dea2a7d0a74e53a357c2d38f52cede is 205, key is testBlockEvictionOnRegionMove1733708724632,,1733708724653.957e404cda065da371c5ed55df4442a7./info:regioninfo/1733708727004/Put/seqid=0 2024-12-09T01:45:27,098 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/info/f6dea2a7d0a74e53a357c2d38f52cede 2024-12-09T01:45:27,107 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c2d646d4b012451a8dc2a28e4d3ff4a2 2024-12-09T01:45:27,121 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/ns/c2d646d4b012451a8dc2a28e4d3ff4a2 is 43, key is default/ns:d/1733708724393/Put/seqid=0 2024-12-09T01:45:27,122 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/ns/c2d646d4b012451a8dc2a28e4d3ff4a2 2024-12-09T01:45:27,130 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 80c07c2f3275490c8ab14298f7cdd766 2024-12-09T01:45:27,143 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/table/80c07c2f3275490c8ab14298f7cdd766 is 78, key is testBlockEvictionOnRegionMove1733708724632/table:state/1733708725072/Put/seqid=0 2024-12-09T01:45:27,144 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/table/80c07c2f3275490c8ab14298f7cdd766 2024-12-09T01:45:27,146 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/info/f6dea2a7d0a74e53a357c2d38f52cede as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/f6dea2a7d0a74e53a357c2d38f52cede 2024-12-09T01:45:27,148 INFO [regionserver/ef6f18c58dc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:27,148 INFO [regionserver/ef6f18c58dc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:27,148 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/f6dea2a7d0a74e53a357c2d38f52cede, delay=986 ms 2024-12-09T01:45:27,149 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/f6dea2a7d0a74e53a357c2d38f52cede, entries=15, sequenceid=15, filesize=7.6 K 2024-12-09T01:45:27,149 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/ns/c2d646d4b012451a8dc2a28e4d3ff4a2 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/c2d646d4b012451a8dc2a28e4d3ff4a2 2024-12-09T01:45:27,151 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/c2d646d4b012451a8dc2a28e4d3ff4a2, delay=919 ms 2024-12-09T01:45:27,152 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/c2d646d4b012451a8dc2a28e4d3ff4a2, entries=2, sequenceid=15, filesize=5.0 K 2024-12-09T01:45:27,152 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/table/80c07c2f3275490c8ab14298f7cdd766 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/80c07c2f3275490c8ab14298f7cdd766 2024-12-09T01:45:27,154 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/80c07c2f3275490c8ab14298f7cdd766, delay=948 ms 2024-12-09T01:45:27,155 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/80c07c2f3275490c8ab14298f7cdd766, entries=2, sequenceid=15, filesize=5.3 K 2024-12-09T01:45:27,156 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=15, compaction requested=false 2024-12-09T01:45:27,157 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T01:45:27,160 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/f6dea2a7d0a74e53a357c2d38f52cede 2024-12-09T01:45:27,162 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/c2d646d4b012451a8dc2a28e4d3ff4a2 2024-12-09T01:45:27,163 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/80c07c2f3275490c8ab14298f7cdd766 2024-12-09T01:45:27,168 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-09T01:45:27,169 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T01:45:27,169 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:45:27,169 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733708727067Running coprocessor pre-close hooks at 1733708727067Disabling compacts and flushes for region at 1733708727067Disabling writes for close at 1733708727067Obtaining lock to block concurrent updates at 1733708727068 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733708727068Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733708727068Flushing stores of hbase:meta,,1.1588230740 at 1733708727069 (+1 ms)Flushing 1588230740/info: creating writer at 1733708727069Flushing 1588230740/info: appending metadata at 1733708727095 (+26 ms)Flushing 1588230740/info: closing flushed file at 1733708727096 (+1 ms)Flushing 1588230740/ns: creating writer at 1733708727101 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733708727120 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1733708727120Flushing 1588230740/table: creating writer at 1733708727124 (+4 ms)Flushing 1588230740/table: appending metadata at 1733708727143 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733708727143Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b6dffe3: reopening flushed file at 1733708727146 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4c4c42f2: reopening flushed file at 1733708727149 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d014756: reopening flushed file at 1733708727152 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=15, compaction requested=false at 1733708727156 (+4 ms)Writing region close event to WAL at 1733708727163 (+7 ms)Running coprocessor post-close hooks at 1733708727169 (+6 ms)Closed at 1733708727169 2024-12-09T01:45:27,170 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T01:45:27,266 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(976): stopping server ef6f18c58dc9,36001,1733708722395; all regions closed. 2024-12-09T01:45:27,267 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(976): stopping server ef6f18c58dc9,40111,1733708722252; all regions closed. 2024-12-09T01:45:27,271 DEBUG [RS:0;ef6f18c58dc9:40111 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs 2024-12-09T01:45:27,271 INFO [RS:0;ef6f18c58dc9:40111 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C40111%2C1733708722252.meta:.meta(num 1733708724258) 2024-12-09T01:45:27,272 DEBUG [RS:1;ef6f18c58dc9:36001 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs 2024-12-09T01:45:27,272 INFO [RS:1;ef6f18c58dc9:36001 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C36001%2C1733708722395:(num 1733708723900) 2024-12-09T01:45:27,272 DEBUG [RS:1;ef6f18c58dc9:36001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:27,272 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:27,273 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:45:27,273 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.ChoreService(370): Chore service for: regionserver/ef6f18c58dc9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T01:45:27,273 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T01:45:27,273 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T01:45:27,273 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T01:45:27,273 INFO [regionserver/ef6f18c58dc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:45:27,273 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:45:27,274 INFO [RS:1;ef6f18c58dc9:36001 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36001 2024-12-09T01:45:27,274 DEBUG [RS:0;ef6f18c58dc9:40111 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs 2024-12-09T01:45:27,274 INFO [RS:0;ef6f18c58dc9:40111 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C40111%2C1733708722252:(num 1733708723901) 2024-12-09T01:45:27,274 DEBUG [RS:0;ef6f18c58dc9:40111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:27,274 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:27,274 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:45:27,274 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.ChoreService(370): Chore service for: regionserver/ef6f18c58dc9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T01:45:27,274 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:45:27,274 INFO [regionserver/ef6f18c58dc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:45:27,275 INFO [RS:0;ef6f18c58dc9:40111 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40111 2024-12-09T01:45:27,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:27,278 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef6f18c58dc9,36001,1733708722395 2024-12-09T01:45:27,279 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:45:27,279 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:45:27,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef6f18c58dc9,40111,1733708722252 2024-12-09T01:45:27,280 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef6f18c58dc9,36001,1733708722395] 2024-12-09T01:45:27,282 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef6f18c58dc9,36001,1733708722395 already deleted, retry=false 2024-12-09T01:45:27,282 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef6f18c58dc9,36001,1733708722395 expired; onlineServers=1 2024-12-09T01:45:27,283 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef6f18c58dc9,40111,1733708722252] 2024-12-09T01:45:27,284 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef6f18c58dc9,40111,1733708722252 already deleted, retry=false 2024-12-09T01:45:27,284 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef6f18c58dc9,40111,1733708722252 expired; onlineServers=0 2024-12-09T01:45:27,284 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef6f18c58dc9,45355,1733708721433' ***** 2024-12-09T01:45:27,284 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T01:45:27,284 INFO [M:0;ef6f18c58dc9:45355 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:45:27,284 INFO [M:0;ef6f18c58dc9:45355 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:45:27,284 DEBUG [M:0;ef6f18c58dc9:45355 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T01:45:27,285 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T01:45:27,285 DEBUG [M:0;ef6f18c58dc9:45355 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T01:45:27,285 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708723515 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708723515,5,FailOnTimeoutGroup] 2024-12-09T01:45:27,285 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708723508 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708723508,5,FailOnTimeoutGroup] 2024-12-09T01:45:27,285 INFO [M:0;ef6f18c58dc9:45355 {}] hbase.ChoreService(370): Chore service for: master/ef6f18c58dc9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T01:45:27,285 INFO [M:0;ef6f18c58dc9:45355 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:45:27,285 DEBUG [M:0;ef6f18c58dc9:45355 {}] master.HMaster(1795): Stopping service threads 2024-12-09T01:45:27,285 INFO [M:0;ef6f18c58dc9:45355 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T01:45:27,285 INFO [M:0;ef6f18c58dc9:45355 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:45:27,286 INFO [M:0;ef6f18c58dc9:45355 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T01:45:27,286 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T01:45:27,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T01:45:27,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,287 DEBUG [M:0;ef6f18c58dc9:45355 {}] zookeeper.ZKUtil(347): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T01:45:27,287 WARN [M:0;ef6f18c58dc9:45355 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T01:45:27,287 INFO [M:0;ef6f18c58dc9:45355 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.lastflushedseqids 2024-12-09T01:45:27,297 INFO [M:0;ef6f18c58dc9:45355 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T01:45:27,297 INFO [M:0;ef6f18c58dc9:45355 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T01:45:27,297 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:45:27,298 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,298 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,298 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:45:27,298 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,298 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-12-09T01:45:27,310 DEBUG [M:0;ef6f18c58dc9:45355 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 18cfedb0c7864e72806cd45740c4d12e 2024-12-09T01:45:27,328 DEBUG [M:0;ef6f18c58dc9:45355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18cfedb0c7864e72806cd45740c4d12e is 82, key is hbase:meta,,1/info:regioninfo/1733708724327/Put/seqid=0 2024-12-09T01:45:27,329 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18cfedb0c7864e72806cd45740c4d12e 2024-12-09T01:45:27,338 DEBUG [M:0;ef6f18c58dc9:45355 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ce9009652dcd480aab44ed65be35cded 2024-12-09T01:45:27,352 DEBUG [M:0;ef6f18c58dc9:45355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ce9009652dcd480aab44ed65be35cded is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733708725076/Put/seqid=0 2024-12-09T01:45:27,353 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ce9009652dcd480aab44ed65be35cded 2024-12-09T01:45:27,363 DEBUG [M:0;ef6f18c58dc9:45355 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 24b5856abf884b6bb2635ccf23ed6627 2024-12-09T01:45:27,376 DEBUG [M:0;ef6f18c58dc9:45355 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24b5856abf884b6bb2635ccf23ed6627 is 69, key is ef6f18c58dc9,36001,1733708722395/rs:state/1733708723650/Put/seqid=0 2024-12-09T01:45:27,377 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24b5856abf884b6bb2635ccf23ed6627 2024-12-09T01:45:27,379 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/18cfedb0c7864e72806cd45740c4d12e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/18cfedb0c7864e72806cd45740c4d12e 2024-12-09T01:45:27,382 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/18cfedb0c7864e72806cd45740c4d12e, entries=8, sequenceid=84, filesize=5.5 K 2024-12-09T01:45:27,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:27,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36001-0x100747364b80002, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:27,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:27,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40111-0x100747364b80001, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:27,382 INFO [RS:0;ef6f18c58dc9:40111 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:45:27,382 INFO [RS:1;ef6f18c58dc9:36001 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:45:27,382 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ce9009652dcd480aab44ed65be35cded as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ce9009652dcd480aab44ed65be35cded 2024-12-09T01:45:27,383 INFO [RS:0;ef6f18c58dc9:40111 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef6f18c58dc9,40111,1733708722252; zookeeper connection closed. 2024-12-09T01:45:27,383 INFO [RS:1;ef6f18c58dc9:36001 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef6f18c58dc9,36001,1733708722395; zookeeper connection closed. 2024-12-09T01:45:27,383 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@12c88779 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@12c88779 2024-12-09T01:45:27,383 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@668bab70 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@668bab70 2024-12-09T01:45:27,383 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-09T01:45:27,385 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ce9009652dcd480aab44ed65be35cded, entries=9, sequenceid=84, filesize=6.5 K 2024-12-09T01:45:27,385 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24b5856abf884b6bb2635ccf23ed6627 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24b5856abf884b6bb2635ccf23ed6627 2024-12-09T01:45:27,388 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24b5856abf884b6bb2635ccf23ed6627, entries=2, sequenceid=84, filesize=5.1 K 2024-12-09T01:45:27,389 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false 2024-12-09T01:45:27,392 INFO [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,392 DEBUG [M:0;ef6f18c58dc9:45355 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733708727297Disabling compacts and flushes for region at 1733708727297Disabling writes for close at 1733708727298 (+1 ms)Obtaining lock to block concurrent updates at 1733708727298Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733708727298Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733708727298Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733708727299 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733708727299Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733708727327 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733708727327Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733708727331 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733708727352 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733708727352Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733708727355 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733708727376 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733708727376Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dad4b5: reopening flushed file at 1733708727379 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da7d62b: reopening flushed file at 1733708727382 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d8bcbb1: reopening flushed file at 1733708727385 (+3 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 90ms, sequenceid=84, compaction requested=false at 1733708727389 (+4 ms)Writing region close event to WAL at 1733708727392 (+3 ms)Closed at 1733708727392 2024-12-09T01:45:27,394 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:45:27,394 INFO [M:0;ef6f18c58dc9:45355 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T01:45:27,395 INFO [M:0;ef6f18c58dc9:45355 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45355 2024-12-09T01:45:27,395 INFO [M:0;ef6f18c58dc9:45355 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:45:27,497 INFO [M:0;ef6f18c58dc9:45355 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:45:27,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:27,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45355-0x100747364b80000, quorum=127.0.0.1:49324, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:27,502 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T01:45:27,523 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-09T01:45:27,523 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,531 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733708726991 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/ef6f18c58dc9:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733708727151 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ef6f18c58dc9:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733708725455 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ef6f18c58dc9:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ef6f18c58dc9:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733708727148 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=330 (was 307) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12023 (was 12376) 2024-12-09T01:45:27,538 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=330, ProcessCount=11, AvailableMemoryMB=12023 2024-12-09T01:45:27,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/hadoop-log-dir so I do NOT create it in target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b 2024-12-09T01:45:27,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/hadoop-tmp-dir so I do NOT create it in target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b 2024-12-09T01:45:27,539 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,542 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/cluster_2aa31df7-2ad8-309f-7bd0-647b24a97dd8, deleteOnExit=true 2024-12-09T01:45:27,545 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/cluster_2aa31df7-2ad8-309f-7bd0-647b24a97dd8/zookeeper_0, clientPort=53028, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/cluster_2aa31df7-2ad8-309f-7bd0-647b24a97dd8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/cluster_2aa31df7-2ad8-309f-7bd0-647b24a97dd8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T01:45:27,546 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53028 2024-12-09T01:45:27,561 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba with version=8 2024-12-09T01:45:27,561 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/hbase-staging 2024-12-09T01:45:27,564 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:45:27,564 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,565 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:45:27,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:45:27,565 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T01:45:27,565 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:45:27,566 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39747 2024-12-09T01:45:27,567 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39747 connecting to ZooKeeper ensemble=127.0.0.1:53028 2024-12-09T01:45:27,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:397470x0, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:45:27,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39747-0x10074737ddf0000 connected 2024-12-09T01:45:27,590 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,590 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:27,592 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba, hbase.cluster.distributed=false 2024-12-09T01:45:27,594 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:45:27,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39747 2024-12-09T01:45:27,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39747 2024-12-09T01:45:27,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39747 2024-12-09T01:45:27,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39747 2024-12-09T01:45:27,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39747 2024-12-09T01:45:27,614 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:45:27,615 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,615 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,615 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:45:27,615 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,615 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:45:27,615 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T01:45:27,615 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:45:27,616 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38179 2024-12-09T01:45:27,618 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38179 connecting to ZooKeeper ensemble=127.0.0.1:53028 2024-12-09T01:45:27,619 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,619 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381790x0, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:45:27,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38179-0x10074737ddf0001 connected 2024-12-09T01:45:27,623 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:27,623 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T01:45:27,624 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.cache 2024-12-09T01:45:27,624 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T01:45:27,625 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T01:45:27,625 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T01:45:27,625 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T01:45:27,625 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T01:45:27,625 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T01:45:27,626 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T01:45:27,626 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T01:45:27,626 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence is complete. 2024-12-09T01:45:27,627 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T01:45:27,628 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:45:27,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38179 2024-12-09T01:45:27,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38179 2024-12-09T01:45:27,636 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38179 2024-12-09T01:45:27,638 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38179 2024-12-09T01:45:27,640 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38179 2024-12-09T01:45:27,656 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:45:27,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,656 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,657 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:45:27,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:27,657 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:45:27,657 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T01:45:27,657 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:45:27,658 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42881 2024-12-09T01:45:27,659 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42881 connecting to ZooKeeper ensemble=127.0.0.1:53028 2024-12-09T01:45:27,659 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,659 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:428810x0, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:45:27,663 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42881-0x10074737ddf0002 connected 2024-12-09T01:45:27,664 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:27,664 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T01:45:27,668 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.cache 2024-12-09T01:45:27,668 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T01:45:27,668 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T01:45:27,669 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T01:45:27,669 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T01:45:27,669 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-09T01:45:27,670 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T01:45:27,670 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence is complete. 2024-12-09T01:45:27,670 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T01:45:27,672 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T01:45:27,673 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T01:45:27,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:45:27,675 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42881 2024-12-09T01:45:27,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42881 2024-12-09T01:45:27,679 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42881 2024-12-09T01:45:27,683 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42881 2024-12-09T01:45:27,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42881 2024-12-09T01:45:27,697 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef6f18c58dc9:39747 2024-12-09T01:45:27,698 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:27,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:27,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:27,700 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:27,701 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:27,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T01:45:27,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T01:45:27,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,705 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T01:45:27,705 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef6f18c58dc9,39747,1733708727564 from backup master directory 2024-12-09T01:45:27,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:27,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:27,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:27,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:45:27,708 WARN [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:45:27,708 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:27,709 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/hbase.id] with ID: 7d6df0b4-47b3-4b31-a4ca-a8d88f9e0ed7 2024-12-09T01:45:27,709 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.tmp/hbase.id 2024-12-09T01:45:27,720 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/hbase.id] 2024-12-09T01:45:27,764 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:27,764 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T01:45:27,766 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-09T01:45:27,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,774 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:45:27,775 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T01:45:27,775 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:27,795 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store 2024-12-09T01:45:27,802 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:27,802 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:45:27,802 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,802 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,803 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:45:27,803 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,803 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:27,803 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733708727802Disabling compacts and flushes for region at 1733708727802Disabling writes for close at 1733708727803 (+1 ms)Writing region close event to WAL at 1733708727803Closed at 1733708727803 2024-12-09T01:45:27,803 WARN [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/.initializing 2024-12-09T01:45:27,803 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/WALs/ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:27,809 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:45:27,812 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef6f18c58dc9%2C39747%2C1733708727564, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/WALs/ef6f18c58dc9,39747,1733708727564, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/oldWALs, maxLogs=10 2024-12-09T01:45:27,831 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/WALs/ef6f18c58dc9,39747,1733708727564/ef6f18c58dc9%2C39747%2C1733708727564.1733708727813 2024-12-09T01:45:27,831 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:27,831 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:27,832 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:27,832 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,832 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,833 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,839 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T01:45:27,839 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,839 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:27,840 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T01:45:27,852 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:27,852 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,857 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T01:45:27,857 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,857 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:27,857 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T01:45:27,862 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,862 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:27,862 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,863 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,863 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,863 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,863 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,863 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:45:27,864 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:45:27,871 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:45:27,871 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70677706, jitterRate=0.05317988991737366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:45:27,872 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733708727832Initializing all the Stores at 1733708727832Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708727833 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708727833Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708727833Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708727833Cleaning up temporary data from old regions at 1733708727863 (+30 ms)Region opened successfully at 1733708727871 (+8 ms) 2024-12-09T01:45:27,873 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T01:45:27,876 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@421eb02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:45:27,877 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T01:45:27,877 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T01:45:27,878 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T01:45:27,878 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T01:45:27,878 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-09T01:45:27,878 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-09T01:45:27,879 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T01:45:27,880 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T01:45:27,881 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T01:45:27,882 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T01:45:27,883 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T01:45:27,884 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T01:45:27,885 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T01:45:27,885 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T01:45:27,886 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T01:45:27,887 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T01:45:27,888 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T01:45:27,889 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T01:45:27,892 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T01:45:27,893 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T01:45:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,896 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef6f18c58dc9,39747,1733708727564, sessionid=0x10074737ddf0000, setting cluster-up flag (Was=false) 2024-12-09T01:45:27,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,904 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T01:45:27,905 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:27,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:27,913 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T01:45:27,914 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:27,915 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T01:45:27,917 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:27,917 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T01:45:27,917 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T01:45:27,917 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef6f18c58dc9,39747,1733708727564 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef6f18c58dc9:0, corePoolSize=10, maxPoolSize=10 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:45:27,919 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:27,923 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733708757923 2024-12-09T01:45:27,923 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T01:45:27,923 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T01:45:27,923 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T01:45:27,923 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T01:45:27,924 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T01:45:27,924 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T01:45:27,924 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:27,924 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:27,924 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T01:45:27,924 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T01:45:27,924 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T01:45:27,924 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,924 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T01:45:27,925 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T01:45:27,925 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T01:45:27,925 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T01:45:27,925 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708727925,5,FailOnTimeoutGroup] 2024-12-09T01:45:27,925 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708727925,5,FailOnTimeoutGroup] 2024-12-09T01:45:27,925 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:27,925 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T01:45:27,925 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:27,926 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:27,938 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T01:45:27,938 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:27,945 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:27,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T01:45:27,949 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:45:27,949 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:27,950 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T01:45:27,954 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:45:27,954 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:27,954 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T01:45:27,958 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:45:27,958 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:27,959 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T01:45:27,963 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:45:27,963 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:27,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:27,963 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:45:27,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:27,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:27,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:45:27,964 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:45:27,964 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:45:27,965 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:45:27,972 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:45:27,972 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61828220, jitterRate=-0.07868772745132446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:45:27,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733708727945Initializing all the Stores at 1733708727945Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708727945Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708727945Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708727945Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708727945Cleaning up temporary data from old regions at 1733708727964 (+19 ms)Region opened successfully at 1733708727973 (+9 ms) 2024-12-09T01:45:27,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:45:27,973 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:45:27,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:45:27,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:45:27,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:45:27,973 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:45:27,974 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733708727973Disabling compacts and flushes for region at 1733708727973Disabling writes for close at 1733708727973Writing region close event to WAL at 1733708727973Closed at 1733708727973 2024-12-09T01:45:27,975 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:27,975 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T01:45:27,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T01:45:27,976 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:45:27,977 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T01:45:27,987 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(746): ClusterId : 7d6df0b4-47b3-4b31-a4ca-a8d88f9e0ed7 2024-12-09T01:45:27,987 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(746): ClusterId : 7d6df0b4-47b3-4b31-a4ca-a8d88f9e0ed7 2024-12-09T01:45:27,987 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T01:45:27,987 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T01:45:27,991 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T01:45:27,991 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T01:45:27,991 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T01:45:27,991 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T01:45:27,993 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T01:45:27,993 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T01:45:27,994 DEBUG [RS:0;ef6f18c58dc9:38179 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2524170e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:45:27,994 DEBUG [RS:1;ef6f18c58dc9:42881 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a985c37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:45:28,006 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef6f18c58dc9:38179 2024-12-09T01:45:28,006 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T01:45:28,007 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T01:45:28,007 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T01:45:28,007 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,39747,1733708727564 with port=38179, startcode=1733708727614 2024-12-09T01:45:28,008 DEBUG [RS:0;ef6f18c58dc9:38179 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T01:45:28,011 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49881, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T01:45:28,011 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39747 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:28,011 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ef6f18c58dc9:42881 2024-12-09T01:45:28,012 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39747 {}] master.ServerManager(517): Registering regionserver=ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:28,012 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T01:45:28,012 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T01:45:28,012 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T01:45:28,013 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,39747,1733708727564 with port=42881, startcode=1733708727656 2024-12-09T01:45:28,013 DEBUG [RS:1;ef6f18c58dc9:42881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T01:45:28,013 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:28,013 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T01:45:28,014 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T01:45:28,015 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T01:45:28,015 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:28,016 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39747 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,016 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39747 {}] master.ServerManager(517): Registering regionserver=ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,016 DEBUG [RS:0;ef6f18c58dc9:38179 {}] zookeeper.ZKUtil(111): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:28,017 WARN [RS:0;ef6f18c58dc9:38179 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:45:28,017 INFO [RS:0;ef6f18c58dc9:38179 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:28,017 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef6f18c58dc9,38179,1733708727614] 2024-12-09T01:45:28,017 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:28,018 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:28,018 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T01:45:28,018 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T01:45:28,024 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:28,024 DEBUG [RS:1;ef6f18c58dc9:42881 {}] zookeeper.ZKUtil(111): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,025 WARN [RS:1;ef6f18c58dc9:42881 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:45:28,025 INFO [RS:1;ef6f18c58dc9:42881 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:28,025 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef6f18c58dc9,42881,1733708727656] 2024-12-09T01:45:28,025 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,025 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T01:45:28,028 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T01:45:28,029 INFO [RS:0;ef6f18c58dc9:38179 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T01:45:28,029 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,030 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T01:45:28,030 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T01:45:28,030 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,030 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,030 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,031 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,032 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,032 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:28,032 DEBUG [RS:0;ef6f18c58dc9:38179 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:28,033 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,033 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,033 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T01:45:28,033 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,033 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,034 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,034 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,38179,1733708727614-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:45:28,034 INFO [RS:1;ef6f18c58dc9:42881 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T01:45:28,034 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,036 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T01:45:28,036 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T01:45:28,036 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,036 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,036 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:28,037 DEBUG [RS:1;ef6f18c58dc9:42881 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:28,038 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,038 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,038 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,038 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,038 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,039 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,42881,1733708727656-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:45:28,056 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T01:45:28,056 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,38179,1733708727614-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,056 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,056 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.Replication(171): ef6f18c58dc9,38179,1733708727614 started 2024-12-09T01:45:28,061 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T01:45:28,062 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,42881,1733708727656-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,062 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,062 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.Replication(171): ef6f18c58dc9,42881,1733708727656 started 2024-12-09T01:45:28,079 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,079 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1482): Serving as ef6f18c58dc9,38179,1733708727614, RpcServer on ef6f18c58dc9/172.17.0.2:38179, sessionid=0x10074737ddf0001 2024-12-09T01:45:28,080 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T01:45:28,080 DEBUG [RS:0;ef6f18c58dc9:38179 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:28,080 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,38179,1733708727614' 2024-12-09T01:45:28,080 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T01:45:28,081 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T01:45:28,081 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T01:45:28,081 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T01:45:28,081 DEBUG [RS:0;ef6f18c58dc9:38179 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:28,081 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,38179,1733708727614' 2024-12-09T01:45:28,081 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T01:45:28,082 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T01:45:28,083 DEBUG [RS:0;ef6f18c58dc9:38179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T01:45:28,083 INFO [RS:0;ef6f18c58dc9:38179 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T01:45:28,083 INFO [RS:0;ef6f18c58dc9:38179 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T01:45:28,083 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,084 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1482): Serving as ef6f18c58dc9,42881,1733708727656, RpcServer on ef6f18c58dc9/172.17.0.2:42881, sessionid=0x10074737ddf0002 2024-12-09T01:45:28,084 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T01:45:28,084 DEBUG [RS:1;ef6f18c58dc9:42881 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,084 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,42881,1733708727656' 2024-12-09T01:45:28,084 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T01:45:28,084 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T01:45:28,085 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T01:45:28,085 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T01:45:28,085 DEBUG [RS:1;ef6f18c58dc9:42881 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,085 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,42881,1733708727656' 2024-12-09T01:45:28,085 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T01:45:28,085 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T01:45:28,086 DEBUG [RS:1;ef6f18c58dc9:42881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T01:45:28,086 INFO [RS:1;ef6f18c58dc9:42881 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T01:45:28,086 INFO [RS:1;ef6f18c58dc9:42881 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T01:45:28,127 WARN [ef6f18c58dc9:39747 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:45:28,184 INFO [RS:0;ef6f18c58dc9:38179 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:45:28,184 INFO [RS:0;ef6f18c58dc9:38179 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef6f18c58dc9%2C38179%2C1733708727614, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,38179,1733708727614, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs, maxLogs=55 2024-12-09T01:45:28,187 INFO [RS:1;ef6f18c58dc9:42881 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:45:28,187 INFO [RS:1;ef6f18c58dc9:42881 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef6f18c58dc9%2C42881%2C1733708727656, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,42881,1733708727656, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs, maxLogs=55 2024-12-09T01:45:28,205 INFO [RS:0;ef6f18c58dc9:38179 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,38179,1733708727614/ef6f18c58dc9%2C38179%2C1733708727614.1733708728186 2024-12-09T01:45:28,207 DEBUG [RS:0;ef6f18c58dc9:38179 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:28,211 INFO [RS:1;ef6f18c58dc9:42881 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,42881,1733708727656/ef6f18c58dc9%2C42881%2C1733708727656.1733708728188 2024-12-09T01:45:28,214 DEBUG [RS:1;ef6f18c58dc9:42881 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:28,377 DEBUG [ef6f18c58dc9:39747 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-09T01:45:28,378 DEBUG [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(204): Hosts are {ef6f18c58dc9=0} racks are {/default-rack=0} 2024-12-09T01:45:28,379 DEBUG [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T01:45:28,379 DEBUG [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T01:45:28,379 DEBUG [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T01:45:28,379 DEBUG [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T01:45:28,379 INFO [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T01:45:28,379 INFO [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T01:45:28,379 DEBUG [ef6f18c58dc9:39747 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T01:45:28,379 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,380 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef6f18c58dc9,42881,1733708727656, state=OPENING 2024-12-09T01:45:28,382 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T01:45:28,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:28,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:28,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:28,384 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:45:28,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:28,385 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,42881,1733708727656}] 2024-12-09T01:45:28,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:28,385 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:28,538 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T01:45:28,540 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37583, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T01:45:28,543 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T01:45:28,543 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:28,543 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T01:45:28,544 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef6f18c58dc9%2C42881%2C1733708727656.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,42881,1733708727656, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs, maxLogs=55 2024-12-09T01:45:28,563 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,42881,1733708727656/ef6f18c58dc9%2C42881%2C1733708727656.meta.1733708728545.meta 2024-12-09T01:45:28,563 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:28,564 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:28,564 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T01:45:28,564 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T01:45:28,564 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T01:45:28,564 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T01:45:28,564 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:28,565 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T01:45:28,565 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T01:45:28,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-09T01:45:28,567 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:45:28,567 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:28,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:28,567 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-09T01:45:28,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:45:28,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:28,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:28,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-09T01:45:28,569 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:45:28,569 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:28,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:28,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-09T01:45:28,570 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:45:28,570 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:28,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:45:28,570 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:45:28,571 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:28,571 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740 2024-12-09T01:45:28,571 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:45:28,572 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:45:28,572 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:45:28,573 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:45:28,573 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64635543, jitterRate=-0.03685535490512848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:45:28,573 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T01:45:28,574 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733708728565Writing region info on filesystem at 1733708728565Initializing all the Stores at 1733708728565Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708728565Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708728565Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708728565Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708728565Cleaning up temporary data from old regions at 1733708728572 (+7 ms)Running coprocessor post-open hooks at 1733708728573 (+1 ms)Region opened successfully at 1733708728574 (+1 ms) 2024-12-09T01:45:28,575 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733708728537 2024-12-09T01:45:28,578 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T01:45:28,578 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T01:45:28,578 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,579 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef6f18c58dc9,42881,1733708727656, state=OPEN 2024-12-09T01:45:28,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:45:28,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:45:28,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:45:28,581 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:28,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:28,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:28,581 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:45:28,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T01:45:28,584 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,42881,1733708727656 in 196 msec 2024-12-09T01:45:28,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T01:45:28,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 609 msec 2024-12-09T01:45:28,586 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:45:28,586 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T01:45:28,590 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:45:28,590 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,42881,1733708727656, seqNum=-1] 2024-12-09T01:45:28,591 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:45:28,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:45:28,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 681 msec 2024-12-09T01:45:28,598 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733708728598, completionTime=-1 2024-12-09T01:45:28,598 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-09T01:45:28,598 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T01:45:28,601 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-09T01:45:28,601 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733708788601 2024-12-09T01:45:28,601 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733708848601 2024-12-09T01:45:28,601 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-09T01:45:28,601 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,39747,1733708727564-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,601 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,39747,1733708727564-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,601 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,39747,1733708727564-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,602 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef6f18c58dc9:39747, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,602 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,602 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,604 DEBUG [master/ef6f18c58dc9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T01:45:28,606 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.898sec 2024-12-09T01:45:28,607 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T01:45:28,607 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T01:45:28,607 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T01:45:28,607 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T01:45:28,607 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T01:45:28,607 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,39747,1733708727564-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:45:28,607 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,39747,1733708727564-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T01:45:28,608 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T01:45:28,608 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T01:45:28,608 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,39747,1733708727564-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:28,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44e7db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:45:28,688 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,39747,-1 for getting cluster id 2024-12-09T01:45:28,688 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:45:28,690 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '7d6df0b4-47b3-4b31-a4ca-a8d88f9e0ed7' 2024-12-09T01:45:28,691 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:45:28,691 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "7d6df0b4-47b3-4b31-a4ca-a8d88f9e0ed7" 2024-12-09T01:45:28,691 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44ef9405, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:45:28,691 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,39747,-1] 2024-12-09T01:45:28,692 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:45:28,692 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:28,694 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:45:28,695 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc06048, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:45:28,696 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:45:28,697 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,42881,1733708727656, seqNum=-1] 2024-12-09T01:45:28,698 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:45:28,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:45:28,701 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:28,702 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:45:28,703 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:28,703 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5dcc47c2 2024-12-09T01:45:28,703 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:45:28,705 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:45:28,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39747 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733708728702', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:45:28,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39747 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 2024-12-09T01:45:28,708 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:45:28,708 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:28,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39747 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733708728702" procId is: 4 2024-12-09T01:45:28,709 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:45:28,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39747 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:28,720 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bd67d34c8755dec6640d8d82a062db5a, NAME => 'testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733708728702', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:28,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:28,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(1722): Closing bd67d34c8755dec6640d8d82a062db5a, disabling compactions & flushes 2024-12-09T01:45:28,727 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:28,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:28,727 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. after waiting 0 ms 2024-12-09T01:45:28,728 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:28,728 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:28,728 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733708728702-pool-0 {}] regionserver.HRegion(1676): Region close journal for bd67d34c8755dec6640d8d82a062db5a: Waiting for close lock at 1733708728727Disabling compacts and flushes for region at 1733708728727Disabling writes for close at 1733708728727Writing region close event to WAL at 1733708728728 (+1 ms)Closed at 1733708728728 2024-12-09T01:45:28,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:45:28,730 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733708728729"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708728729"}]},"ts":"1733708728729"} 2024-12-09T01:45:28,732 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:45:28,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:45:28,734 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733708728702","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708728734"}]},"ts":"1733708728734"} 2024-12-09T01:45:28,737 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733708728702, state=ENABLING in hbase:meta 2024-12-09T01:45:28,737 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ef6f18c58dc9=0} racks are {/default-rack=0} 2024-12-09T01:45:28,738 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-09T01:45:28,738 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-09T01:45:28,738 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-09T01:45:28,738 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-09T01:45:28,738 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-09T01:45:28,738 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-09T01:45:28,738 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-09T01:45:28,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN}] 2024-12-09T01:45:28,740 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN 2024-12-09T01:45:28,740 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,38179,1733708727614; forceNewPlan=false, retain=false 2024-12-09T01:45:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39747 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:28,891 INFO [ef6f18c58dc9:39747 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-09T01:45:28,891 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bd67d34c8755dec6640d8d82a062db5a, regionState=OPENING, regionLocation=ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:28,894 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN because future has completed 2024-12-09T01:45:28,895 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd67d34c8755dec6640d8d82a062db5a, server=ef6f18c58dc9,38179,1733708727614}] 2024-12-09T01:45:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39747 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:29,048 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T01:45:29,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57473, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T01:45:29,052 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:29,052 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bd67d34c8755dec6640d8d82a062db5a, NAME => 'testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:29,053 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733708728702 bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,053 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:29,053 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,053 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,054 INFO [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,058 INFO [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd67d34c8755dec6640d8d82a062db5a columnFamilyName family 2024-12-09T01:45:29,058 DEBUG [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:29,059 INFO [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] regionserver.HStore(327): Store=bd67d34c8755dec6640d8d82a062db5a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:29,059 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,059 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,059 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,060 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,060 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,061 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,070 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:45:29,070 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bd67d34c8755dec6640d8d82a062db5a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71196850, jitterRate=0.060915738344192505}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:45:29,070 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:29,071 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bd67d34c8755dec6640d8d82a062db5a: Running coprocessor pre-open hook at 1733708729053Writing region info on filesystem at 1733708729053Initializing all the Stores at 1733708729054 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708729054Cleaning up temporary data from old regions at 1733708729060 (+6 ms)Running coprocessor post-open hooks at 1733708729070 (+10 ms)Region opened successfully at 1733708729071 (+1 ms) 2024-12-09T01:45:29,071 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a., pid=6, masterSystemTime=1733708729047 2024-12-09T01:45:29,074 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:29,074 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:29,075 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bd67d34c8755dec6640d8d82a062db5a, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:29,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd67d34c8755dec6640d8d82a062db5a, server=ef6f18c58dc9,38179,1733708727614 because future has completed 2024-12-09T01:45:29,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T01:45:29,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bd67d34c8755dec6640d8d82a062db5a, server=ef6f18c58dc9,38179,1733708727614 in 183 msec 2024-12-09T01:45:29,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T01:45:29,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN in 342 msec 2024-12-09T01:45:29,083 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:45:29,084 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733708728702","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708729083"}]},"ts":"1733708729083"} 2024-12-09T01:45:29,086 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733708728702, state=ENABLED in hbase:meta 2024-12-09T01:45:29,087 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:45:29,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733708728702 in 381 msec 2024-12-09T01:45:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39747 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:45:29,339 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733708728702 completed 2024-12-09T01:45:29,339 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733708728702 get assigned. Timeout = 60000ms 2024-12-09T01:45:29,339 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:45:29,343 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733708728702 assigned to meta. Checking AM states. 2024-12-09T01:45:29,343 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-09T01:45:29,344 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733708728702 assigned. 2024-12-09T01:45:29,347 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733708728702', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a., hostname=ef6f18c58dc9,38179,1733708727614, seqNum=2] 2024-12-09T01:45:29,348 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:45:29,350 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:45:29,354 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bd67d34c8755dec6640d8d82a062db5a 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-09T01:45:29,365 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4e0ccd034b3c42a88e1ad7d2aa6bce06 2024-12-09T01:45:29,379 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/.tmp/family/4e0ccd034b3c42a88e1ad7d2aa6bce06 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-09T01:45:29,379 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/.tmp/family/4e0ccd034b3c42a88e1ad7d2aa6bce06 2024-12-09T01:45:29,382 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/.tmp/family/4e0ccd034b3c42a88e1ad7d2aa6bce06 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06 2024-12-09T01:45:29,383 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06, delay=995 ms 2024-12-09T01:45:29,384 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06, entries=2, sequenceid=6, filesize=4.9 K 2024-12-09T01:45:29,385 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for bd67d34c8755dec6640d8d82a062db5a in 31ms, sequenceid=6, compaction requested=false 2024-12-09T01:45:29,385 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733708728702' 2024-12-09T01:45:29,386 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bd67d34c8755dec6640d8d82a062db5a: 2024-12-09T01:45:29,796 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:45:30,379 DEBUG [hfile-prefetch-1733708726991 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4e0ccd034b3c42a88e1ad7d2aa6bce06_0. 2024-12-09T01:45:30,380 DEBUG [hfile-prefetch-1733708726991 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4e0ccd034b3c42a88e1ad7d2aa6bce06_117. 2024-12-09T01:45:30,380 DEBUG [hfile-prefetch-1733708726991 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06, with total blocks 2, and data blocks 1 2024-12-09T01:45:30,380 DEBUG [hfile-prefetch-1733708726991 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-09T01:45:30,380 DEBUG [hfile-prefetch-1733708726991 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-09T01:45:30,380 DEBUG [hfile-prefetch-1733708726991 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06. Will sleep 100ms and try the verification again. 2024-12-09T01:45:30,386 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;ef6f18c58dc9:38179,5,FailOnTimeoutGroup] 2024-12-09T01:45:30,386 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef6f18c58dc9,38179,1733708727614' ***** 2024-12-09T01:45:30,386 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-09T01:45:30,387 INFO [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T01:45:30,387 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T01:45:30,387 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T01:45:30,387 INFO [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence 2024-12-09T01:45:30,387 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T01:45:30,387 INFO [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T01:45:30,387 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T01:45:30,397 DEBUG [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1487): Thread RS:0;ef6f18c58dc9:38179 started persisting bucket cache to file 2024-12-09T01:45:30,397 DEBUG [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:30,397 DEBUG [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:30,404 DEBUG [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-09T01:45:30,405 DEBUG [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733708730397 2024-12-09T01:45:30,405 DEBUG [RS:0;ef6f18c58dc9:38179 {}] bucket.BucketCache(1509): Thread RS:0;ef6f18c58dc9:38179 finished persisting bucket cache to file, renaming 2024-12-09T01:45:30,405 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T01:45:30,405 INFO [RS:0;ef6f18c58dc9:38179 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T01:45:30,405 INFO [RS:0;ef6f18c58dc9:38179 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T01:45:30,405 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T01:45:30,405 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(3091): Received CLOSE for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,405 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(959): stopping server ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:30,405 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:45:30,405 INFO [RS:0;ef6f18c58dc9:38179 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef6f18c58dc9:38179. 2024-12-09T01:45:30,406 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T01:45:30,406 DEBUG [RS:0;ef6f18c58dc9:38179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:30,406 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd67d34c8755dec6640d8d82a062db5a, disabling compactions & flushes 2024-12-09T01:45:30,406 DEBUG [RS:0;ef6f18c58dc9:38179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:30,406 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,406 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,406 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. after waiting 0 ms 2024-12-09T01:45:30,406 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,406 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T01:45:30,406 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1325): Online Regions={bd67d34c8755dec6640d8d82a062db5a=testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.} 2024-12-09T01:45:30,406 DEBUG [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1351): Waiting on bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,406 DEBUG [StoreFileCloser-bd67d34c8755dec6640d8d82a062db5a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06 2024-12-09T01:45:30,409 WARN [hfile-prefetch-1733708726991 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-09T01:45:30,409 DEBUG [hfile-prefetch-1733708726991 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4e0ccd034b3c42a88e1ad7d2aa6bce06 2024-12-09T01:45:30,411 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-09T01:45:30,412 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,412 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd67d34c8755dec6640d8d82a062db5a: Waiting for close lock at 1733708730405Running coprocessor pre-close hooks at 1733708730406 (+1 ms)Disabling compacts and flushes for region at 1733708730406Disabling writes for close at 1733708730406Writing region close event to WAL at 1733708730407 (+1 ms)Running coprocessor post-close hooks at 1733708730412 (+5 ms)Closed at 1733708730412 2024-12-09T01:45:30,412 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,436 INFO [regionserver/ef6f18c58dc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:30,471 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-09T01:45:30,471 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-09T01:45:30,471 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:30,471 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:30,475 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:30,475 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708730471 2024-12-09T01:45:30,475 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-09T01:45:30,606 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(976): stopping server ef6f18c58dc9,38179,1733708727614; all regions closed. 2024-12-09T01:45:30,608 DEBUG [RS:0;ef6f18c58dc9:38179 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs 2024-12-09T01:45:30,608 INFO [RS:0;ef6f18c58dc9:38179 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C38179%2C1733708727614:(num 1733708728186) 2024-12-09T01:45:30,608 DEBUG [RS:0;ef6f18c58dc9:38179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:30,608 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:30,609 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:45:30,609 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.ChoreService(370): Chore service for: regionserver/ef6f18c58dc9:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T01:45:30,609 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T01:45:30,609 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T01:45:30,609 INFO [regionserver/ef6f18c58dc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:45:30,609 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T01:45:30,609 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:45:30,609 INFO [RS:0;ef6f18c58dc9:38179 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38179 2024-12-09T01:45:30,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:30,614 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef6f18c58dc9,38179,1733708727614 2024-12-09T01:45:30,614 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:45:30,615 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef6f18c58dc9,38179,1733708727614] 2024-12-09T01:45:30,617 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef6f18c58dc9,38179,1733708727614 already deleted, retry=false 2024-12-09T01:45:30,617 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ef6f18c58dc9,38179,1733708727614 on ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:30,622 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ef6f18c58dc9,38179,1733708727614, splitWal=true, meta=false 2024-12-09T01:45:30,624 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for ef6f18c58dc9,38179,1733708727614 (carryingMeta=false) ef6f18c58dc9,38179,1733708727614/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4389036b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-09T01:45:30,624 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ef6f18c58dc9,38179,1733708727614, splitWal=true, meta=false 2024-12-09T01:45:30,626 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): ef6f18c58dc9,38179,1733708727614 had 1 regions 2024-12-09T01:45:30,627 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ef6f18c58dc9,38179,1733708727614, splitWal=true, meta=false, isMeta: false 2024-12-09T01:45:30,627 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,38179,1733708727614-splitting 2024-12-09T01:45:30,628 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,38179,1733708727614-splitting dir is empty, no logs to split. 2024-12-09T01:45:30,628 INFO [PEWorker-2 {}] master.SplitWALManager(105): ef6f18c58dc9,38179,1733708727614 WAL count=0, meta=false 2024-12-09T01:45:30,630 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,38179,1733708727614-splitting dir is empty, no logs to split. 2024-12-09T01:45:30,630 INFO [PEWorker-2 {}] master.SplitWALManager(105): ef6f18c58dc9,38179,1733708727614 WAL count=0, meta=false 2024-12-09T01:45:30,630 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ef6f18c58dc9,38179,1733708727614 WAL splitting is done? wals=0, meta=false 2024-12-09T01:45:30,630 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for ef6f18c58dc9,38179,1733708727614 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,38179,1733708727614-splitting does not exist 2024-12-09T01:45:30,632 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN}] 2024-12-09T01:45:30,633 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN 2024-12-09T01:45:30,634 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-09T01:45:30,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:30,716 INFO [RS:0;ef6f18c58dc9:38179 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:45:30,716 INFO [RS:0;ef6f18c58dc9:38179 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef6f18c58dc9,38179,1733708727614; zookeeper connection closed. 2024-12-09T01:45:30,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38179-0x10074737ddf0001, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:30,716 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@23f8413e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@23f8413e 2024-12-09T01:45:30,785 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bd67d34c8755dec6640d8d82a062db5a, regionState=OPENING, regionLocation=ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:30,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN because future has completed 2024-12-09T01:45:30,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd67d34c8755dec6640d8d82a062db5a, server=ef6f18c58dc9,42881,1733708727656}] 2024-12-09T01:45:30,903 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:45:30,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:30,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:30,903 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:45:30,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:45:30,903 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:45:30,903 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T01:45:30,904 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:45:30,904 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36067 2024-12-09T01:45:30,906 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36067 connecting to ZooKeeper ensemble=127.0.0.1:53028 2024-12-09T01:45:30,907 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:30,907 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:30,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360670x0, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:45:30,910 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36067-0x10074737ddf0003 connected 2024-12-09T01:45:30,910 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-09T01:45:30,910 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-09T01:45:30,911 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T01:45:30,914 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.cache 2024-12-09T01:45:30,914 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-09T01:45:30,914 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-09T01:45:30,914 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-09T01:45:30,914 INFO [Thread-200 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-09T01:45:30,914 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-09T01:45:30,915 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T01:45:30,915 INFO [Thread-200 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-09T01:45:30,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T01:45:30,916 INFO [Thread-200 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-09T01:45:30,917 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:45:30,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36067 2024-12-09T01:45:30,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36067 2024-12-09T01:45:30,918 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36067 2024-12-09T01:45:30,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36067 2024-12-09T01:45:30,919 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36067 2024-12-09T01:45:30,919 DEBUG [Thread-200 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-09T01:45:30,920 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-09T01:45:30,920 INFO [Thread-200 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-09T01:45:30,920 INFO [Thread-200 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence is complete. 2024-12-09T01:45:30,921 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(746): ClusterId : 7d6df0b4-47b3-4b31-a4ca-a8d88f9e0ed7 2024-12-09T01:45:30,921 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T01:45:30,923 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T01:45:30,923 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T01:45:30,925 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T01:45:30,926 DEBUG [RS:2;ef6f18c58dc9:36067 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@95cabf5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:45:30,938 DEBUG [RS:2;ef6f18c58dc9:36067 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ef6f18c58dc9:36067 2024-12-09T01:45:30,938 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T01:45:30,938 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T01:45:30,938 DEBUG [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T01:45:30,939 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,39747,1733708727564 with port=36067, startcode=1733708730903 2024-12-09T01:45:30,939 DEBUG [RS:2;ef6f18c58dc9:36067 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T01:45:30,941 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57575, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T01:45:30,942 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39747 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:30,942 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39747 {}] master.ServerManager(517): Registering regionserver=ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:30,943 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,943 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => bd67d34c8755dec6640d8d82a062db5a, NAME => 'testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:45:30,943 DEBUG [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba 2024-12-09T01:45:30,944 DEBUG [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-09T01:45:30,944 DEBUG [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T01:45:30,944 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733708728702 bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,944 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:45:30,944 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,944 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,945 INFO [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,945 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:30,946 INFO [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bd67d34c8755dec6640d8d82a062db5a columnFamilyName family 2024-12-09T01:45:30,946 DEBUG [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:45:30,946 DEBUG [RS:2;ef6f18c58dc9:36067 {}] zookeeper.ZKUtil(111): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:30,946 WARN [RS:2;ef6f18c58dc9:36067 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:45:30,946 INFO [RS:2;ef6f18c58dc9:36067 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:45:30,946 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef6f18c58dc9,36067,1733708730903] 2024-12-09T01:45:30,946 DEBUG [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:30,949 DEBUG [StoreFileOpener-bd67d34c8755dec6640d8d82a062db5a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06, delay=961 ms 2024-12-09T01:45:30,950 DEBUG [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06 2024-12-09T01:45:30,950 INFO [StoreOpener-bd67d34c8755dec6640d8d82a062db5a-1 {}] regionserver.HStore(327): Store=bd67d34c8755dec6640d8d82a062db5a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:45:30,950 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,951 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,951 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,951 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,952 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,953 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T01:45:30,953 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,954 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened bd67d34c8755dec6640d8d82a062db5a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67225484, jitterRate=0.0017377734184265137}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T01:45:30,954 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:30,956 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for bd67d34c8755dec6640d8d82a062db5a: Running coprocessor pre-open hook at 1733708730944Writing region info on filesystem at 1733708730944Initializing all the Stores at 1733708730945 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708730945Cleaning up temporary data from old regions at 1733708730952 (+7 ms)Running coprocessor post-open hooks at 1733708730954 (+2 ms)Region opened successfully at 1733708730955 (+1 ms) 2024-12-09T01:45:30,956 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a., pid=9, masterSystemTime=1733708730940 2024-12-09T01:45:30,956 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T01:45:30,958 INFO [RS:2;ef6f18c58dc9:36067 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T01:45:30,958 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,958 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T01:45:30,959 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T01:45:30,959 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,959 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:30,959 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bd67d34c8755dec6640d8d82a062db5a, regionState=OPEN, openSeqNum=10, regionLocation=ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:30,960 DEBUG [RS:2;ef6f18c58dc9:36067 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:45:30,962 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,962 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,962 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,962 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,962 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,962 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,36067,1733708730903-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:45:30,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bd67d34c8755dec6640d8d82a062db5a, server=ef6f18c58dc9,42881,1733708727656 because future has completed 2024-12-09T01:45:30,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-09T01:45:30,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure bd67d34c8755dec6640d8d82a062db5a, server=ef6f18c58dc9,42881,1733708727656 in 181 msec 2024-12-09T01:45:30,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T01:45:30,975 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733708728702, region=bd67d34c8755dec6640d8d82a062db5a, ASSIGN in 340 msec 2024-12-09T01:45:30,975 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server ef6f18c58dc9,38179,1733708727614 after splitting done 2024-12-09T01:45:30,977 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure ef6f18c58dc9,38179,1733708727614, splitWal=true, meta=false in 357 msec 2024-12-09T01:45:30,981 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T01:45:30,981 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,36067,1733708730903-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,981 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,981 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.Replication(171): ef6f18c58dc9,36067,1733708730903 started 2024-12-09T01:45:30,999 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:45:30,999 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(1482): Serving as ef6f18c58dc9,36067,1733708730903, RpcServer on ef6f18c58dc9/172.17.0.2:36067, sessionid=0x10074737ddf0003 2024-12-09T01:45:30,999 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T01:45:30,999 DEBUG [RS:2;ef6f18c58dc9:36067 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:30,999 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,36067,1733708730903' 2024-12-09T01:45:30,999 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T01:45:31,000 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T01:45:31,000 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T01:45:31,001 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T01:45:31,001 DEBUG [RS:2;ef6f18c58dc9:36067 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:31,001 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,36067,1733708730903' 2024-12-09T01:45:31,001 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T01:45:31,001 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T01:45:31,002 DEBUG [RS:2;ef6f18c58dc9:36067 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T01:45:31,002 INFO [RS:2;ef6f18c58dc9:36067 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T01:45:31,002 INFO [RS:2;ef6f18c58dc9:36067 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T01:45:31,102 INFO [RS:2;ef6f18c58dc9:36067 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:45:31,103 INFO [RS:2;ef6f18c58dc9:36067 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef6f18c58dc9%2C36067%2C1733708730903, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,36067,1733708730903, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs, maxLogs=55 2024-12-09T01:45:31,124 INFO [RS:2;ef6f18c58dc9:36067 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/WALs/ef6f18c58dc9,36067,1733708730903/ef6f18c58dc9%2C36067%2C1733708730903.1733708731104 2024-12-09T01:45:31,125 DEBUG [RS:2;ef6f18c58dc9:36067 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-09T01:45:31,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T01:45:31,499 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:45:31,499 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:31,499 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:31,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:31,500 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:45:31,500 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T01:45:31,500 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=879228482, stopped=false 2024-12-09T01:45:31,500 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef6f18c58dc9,39747,1733708727564 2024-12-09T01:45:31,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:31,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:31,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:31,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:45:31,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:31,502 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:45:31,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:31,503 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:45:31,503 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:31,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:31,503 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef6f18c58dc9,42881,1733708727656' ***** 2024-12-09T01:45:31,503 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T01:45:31,503 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef6f18c58dc9,36067,1733708730903' ***** 2024-12-09T01:45:31,503 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T01:45:31,503 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:31,504 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:31,504 INFO [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T01:45:31,504 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:45:31,504 INFO [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence 2024-12-09T01:45:31,504 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T01:45:31,504 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T01:45:31,504 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T01:45:31,504 INFO [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T01:45:31,504 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T01:45:31,505 INFO [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1738): Disabling cache 2024-12-09T01:45:31,506 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-09T01:45:31,506 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-09T01:45:31,506 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-09T01:45:31,506 INFO [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/054734d4-a5da-f281-f7c4-28ffd786e95b/bucket.persistence 2024-12-09T01:45:31,506 INFO [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-09T01:45:31,506 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-09T01:45:31,514 DEBUG [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1487): Thread RS:2;ef6f18c58dc9:36067 started persisting bucket cache to file 2024-12-09T01:45:31,515 DEBUG [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:31,515 DEBUG [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:31,516 DEBUG [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1487): Thread RS:1;ef6f18c58dc9:42881 started persisting bucket cache to file 2024-12-09T01:45:31,516 DEBUG [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-09T01:45:31,516 DEBUG [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-09T01:45:31,518 DEBUG [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:31,518 DEBUG [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708731514 2024-12-09T01:45:31,518 DEBUG [RS:2;ef6f18c58dc9:36067 {}] bucket.BucketCache(1509): Thread RS:2;ef6f18c58dc9:36067 finished persisting bucket cache to file, renaming 2024-12-09T01:45:31,518 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T01:45:31,519 INFO [RS:2;ef6f18c58dc9:36067 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T01:45:31,519 INFO [RS:2;ef6f18c58dc9:36067 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T01:45:31,519 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T01:45:31,519 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(959): stopping server ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:31,519 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:45:31,519 INFO [RS:2;ef6f18c58dc9:36067 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ef6f18c58dc9:36067. 2024-12-09T01:45:31,519 DEBUG [RS:2;ef6f18c58dc9:36067 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:31,519 DEBUG [RS:2;ef6f18c58dc9:36067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:31,519 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(976): stopping server ef6f18c58dc9,36067,1733708730903; all regions closed. 2024-12-09T01:45:31,519 DEBUG [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-09T01:45:31,519 DEBUG [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733708731516 2024-12-09T01:45:31,519 DEBUG [RS:1;ef6f18c58dc9:42881 {}] bucket.BucketCache(1509): Thread RS:1;ef6f18c58dc9:42881 finished persisting bucket cache to file, renaming 2024-12-09T01:45:31,520 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T01:45:31,520 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T01:45:31,520 INFO [RS:1;ef6f18c58dc9:42881 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T01:45:31,520 INFO [RS:1;ef6f18c58dc9:42881 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T01:45:31,520 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(3091): Received CLOSE for bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:31,520 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(959): stopping server ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:31,520 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:45:31,521 INFO [RS:1;ef6f18c58dc9:42881 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ef6f18c58dc9:42881. 2024-12-09T01:45:31,521 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T01:45:31,521 DEBUG [RS:1;ef6f18c58dc9:42881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:45:31,521 DEBUG [RS:1;ef6f18c58dc9:42881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:31,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bd67d34c8755dec6640d8d82a062db5a, disabling compactions & flushes 2024-12-09T01:45:31,521 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:31,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:31,521 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T01:45:31,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. after waiting 0 ms 2024-12-09T01:45:31,521 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T01:45:31,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:31,521 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T01:45:31,521 DEBUG [RS:2;ef6f18c58dc9:36067 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs 2024-12-09T01:45:31,521 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T01:45:31,521 INFO [RS:2;ef6f18c58dc9:36067 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C36067%2C1733708730903:(num 1733708731104) 2024-12-09T01:45:31,521 DEBUG [RS:2;ef6f18c58dc9:36067 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:31,521 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:31,521 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:45:31,521 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-09T01:45:31,522 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1325): Online Regions={bd67d34c8755dec6640d8d82a062db5a=testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a., 1588230740=hbase:meta,,1.1588230740} 2024-12-09T01:45:31,522 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-09T01:45:31,522 DEBUG [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, bd67d34c8755dec6640d8d82a062db5a 2024-12-09T01:45:31,522 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:45:31,522 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.ChoreService(370): Chore service for: regionserver/ef6f18c58dc9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-09T01:45:31,522 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:45:31,522 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:45:31,522 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:45:31,522 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T01:45:31,522 INFO [regionserver/ef6f18c58dc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:45:31,522 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:45:31,522 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T01:45:31,522 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T01:45:31,522 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:45:31,522 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-09T01:45:31,522 DEBUG [StoreFileCloser-bd67d34c8755dec6640d8d82a062db5a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/family/4e0ccd034b3c42a88e1ad7d2aa6bce06 2024-12-09T01:45:31,522 INFO [RS:2;ef6f18c58dc9:36067 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36067 2024-12-09T01:45:31,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:31,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef6f18c58dc9,36067,1733708730903 2024-12-09T01:45:31,525 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:45:31,526 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef6f18c58dc9,36067,1733708730903] 2024-12-09T01:45:31,528 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef6f18c58dc9,36067,1733708730903 already deleted, retry=false 2024-12-09T01:45:31,528 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef6f18c58dc9,36067,1733708730903 expired; onlineServers=1 2024-12-09T01:45:31,528 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/default/testBlockEvictionOnGracefulStop1733708728702/bd67d34c8755dec6640d8d82a062db5a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-09T01:45:31,528 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:31,528 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bd67d34c8755dec6640d8d82a062db5a: Waiting for close lock at 1733708731520Running coprocessor pre-close hooks at 1733708731521 (+1 ms)Disabling compacts and flushes for region at 1733708731521Disabling writes for close at 1733708731521Writing region close event to WAL at 1733708731523 (+2 ms)Running coprocessor post-close hooks at 1733708731528 (+5 ms)Closed at 1733708731528 2024-12-09T01:45:31,529 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a. 2024-12-09T01:45:31,534 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d00d25efa184be7a311d00ffee7d80d 2024-12-09T01:45:31,541 INFO [regionserver/ef6f18c58dc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:31,547 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/info/4d00d25efa184be7a311d00ffee7d80d is 209, key is testBlockEvictionOnGracefulStop1733708728702,,1733708728706.bd67d34c8755dec6640d8d82a062db5a./info:regioninfo/1733708730960/Put/seqid=0 2024-12-09T01:45:31,548 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/info/4d00d25efa184be7a311d00ffee7d80d 2024-12-09T01:45:31,557 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on caf334d4c8804e1d857afe1ed0763681 2024-12-09T01:45:31,564 INFO [regionserver/ef6f18c58dc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:31,570 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/ns/caf334d4c8804e1d857afe1ed0763681 is 43, key is default/ns:d/1733708728593/Put/seqid=0 2024-12-09T01:45:31,571 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/ns/caf334d4c8804e1d857afe1ed0763681 2024-12-09T01:45:31,579 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9422988fbc664183bca8f81272f451e9 2024-12-09T01:45:31,593 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/table/9422988fbc664183bca8f81272f451e9 is 80, key is testBlockEvictionOnGracefulStop1733708728702/table:state/1733708729083/Put/seqid=0 2024-12-09T01:45:31,593 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/table/9422988fbc664183bca8f81272f451e9 2024-12-09T01:45:31,596 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/info/4d00d25efa184be7a311d00ffee7d80d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/4d00d25efa184be7a311d00ffee7d80d 2024-12-09T01:45:31,597 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/4d00d25efa184be7a311d00ffee7d80d, delay=904 ms 2024-12-09T01:45:31,598 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/4d00d25efa184be7a311d00ffee7d80d, entries=14, sequenceid=13, filesize=7.5 K 2024-12-09T01:45:31,598 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/ns/caf334d4c8804e1d857afe1ed0763681 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/caf334d4c8804e1d857afe1ed0763681 2024-12-09T01:45:31,599 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/caf334d4c8804e1d857afe1ed0763681, delay=917 ms 2024-12-09T01:45:31,600 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/caf334d4c8804e1d857afe1ed0763681, entries=2, sequenceid=13, filesize=5.0 K 2024-12-09T01:45:31,600 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/.tmp/table/9422988fbc664183bca8f81272f451e9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/9422988fbc664183bca8f81272f451e9 2024-12-09T01:45:31,602 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/9422988fbc664183bca8f81272f451e9, delay=926 ms 2024-12-09T01:45:31,602 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/9422988fbc664183bca8f81272f451e9, entries=2, sequenceid=13, filesize=5.3 K 2024-12-09T01:45:31,603 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false 2024-12-09T01:45:31,604 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/info/4d00d25efa184be7a311d00ffee7d80d 2024-12-09T01:45:31,605 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/ns/caf334d4c8804e1d857afe1ed0763681 2024-12-09T01:45:31,605 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/table/9422988fbc664183bca8f81272f451e9 2024-12-09T01:45:31,610 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-09T01:45:31,610 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T01:45:31,610 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:45:31,610 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733708731522Running coprocessor pre-close hooks at 1733708731522Disabling compacts and flushes for region at 1733708731522Disabling writes for close at 1733708731522Obtaining lock to block concurrent updates at 1733708731522Preparing flush snapshotting stores in 1588230740 at 1733708731522Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733708731523 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733708731523Flushing 1588230740/info: creating writer at 1733708731523Flushing 1588230740/info: appending metadata at 1733708731547 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733708731547Flushing 1588230740/ns: creating writer at 1733708731550 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733708731570 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1733708731570Flushing 1588230740/table: creating writer at 1733708731573 (+3 ms)Flushing 1588230740/table: appending metadata at 1733708731592 (+19 ms)Flushing 1588230740/table: closing flushed file at 1733708731592Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@610a4500: reopening flushed file at 1733708731595 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d3e0bcf: reopening flushed file at 1733708731598 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71a59e89: reopening flushed file at 1733708731600 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 81ms, sequenceid=13, compaction requested=false at 1733708731603 (+3 ms)Writing region close event to WAL at 1733708731605 (+2 ms)Running coprocessor post-close hooks at 1733708731610 (+5 ms)Closed at 1733708731610 2024-12-09T01:45:31,610 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T01:45:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:31,627 INFO [RS:2;ef6f18c58dc9:36067 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:45:31,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36067-0x10074737ddf0003, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:31,627 INFO [RS:2;ef6f18c58dc9:36067 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef6f18c58dc9,36067,1733708730903; zookeeper connection closed. 2024-12-09T01:45:31,627 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75f81f9d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75f81f9d 2024-12-09T01:45:31,722 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(976): stopping server ef6f18c58dc9,42881,1733708727656; all regions closed. 2024-12-09T01:45:31,724 DEBUG [RS:1;ef6f18c58dc9:42881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs 2024-12-09T01:45:31,724 INFO [RS:1;ef6f18c58dc9:42881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C42881%2C1733708727656.meta:.meta(num 1733708728545) 2024-12-09T01:45:31,726 DEBUG [RS:1;ef6f18c58dc9:42881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/oldWALs 2024-12-09T01:45:31,726 INFO [RS:1;ef6f18c58dc9:42881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C42881%2C1733708727656:(num 1733708728188) 2024-12-09T01:45:31,726 DEBUG [RS:1;ef6f18c58dc9:42881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:45:31,726 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:45:31,726 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:45:31,727 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.ChoreService(370): Chore service for: regionserver/ef6f18c58dc9:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T01:45:31,727 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:45:31,727 INFO [regionserver/ef6f18c58dc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:45:31,727 INFO [RS:1;ef6f18c58dc9:42881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42881 2024-12-09T01:45:31,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef6f18c58dc9,42881,1733708727656 2024-12-09T01:45:31,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:45:31,729 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:45:31,731 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef6f18c58dc9,42881,1733708727656] 2024-12-09T01:45:31,734 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef6f18c58dc9,42881,1733708727656 already deleted, retry=false 2024-12-09T01:45:31,734 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef6f18c58dc9,42881,1733708727656 expired; onlineServers=0 2024-12-09T01:45:31,734 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef6f18c58dc9,39747,1733708727564' ***** 2024-12-09T01:45:31,734 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T01:45:31,734 INFO [M:0;ef6f18c58dc9:39747 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:45:31,734 INFO [M:0;ef6f18c58dc9:39747 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:45:31,734 DEBUG [M:0;ef6f18c58dc9:39747 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T01:45:31,734 DEBUG [M:0;ef6f18c58dc9:39747 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T01:45:31,734 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T01:45:31,734 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708727925 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708727925,5,FailOnTimeoutGroup] 2024-12-09T01:45:31,734 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708727925 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708727925,5,FailOnTimeoutGroup] 2024-12-09T01:45:31,734 INFO [M:0;ef6f18c58dc9:39747 {}] hbase.ChoreService(370): Chore service for: master/ef6f18c58dc9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T01:45:31,735 INFO [M:0;ef6f18c58dc9:39747 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:45:31,735 DEBUG [M:0;ef6f18c58dc9:39747 {}] master.HMaster(1795): Stopping service threads 2024-12-09T01:45:31,735 INFO [M:0;ef6f18c58dc9:39747 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T01:45:31,735 INFO [M:0;ef6f18c58dc9:39747 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:45:31,735 INFO [M:0;ef6f18c58dc9:39747 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T01:45:31,735 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T01:45:31,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T01:45:31,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:45:31,738 DEBUG [M:0;ef6f18c58dc9:39747 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-12-09T01:45:31,738 DEBUG [M:0;ef6f18c58dc9:39747 {}] master.ActiveMasterManager(353): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-09T01:45:31,738 INFO [M:0;ef6f18c58dc9:39747 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/.lastflushedseqids 2024-12-09T01:45:31,746 INFO [M:0;ef6f18c58dc9:39747 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T01:45:31,746 INFO [M:0;ef6f18c58dc9:39747 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T01:45:31,746 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:45:31,746 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:31,746 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:31,746 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:45:31,746 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:31,747 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-12-09T01:45:31,757 DEBUG [M:0;ef6f18c58dc9:39747 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0dc1c6dbe5ca47f998dbd8ca60408c1c 2024-12-09T01:45:31,774 DEBUG [M:0;ef6f18c58dc9:39747 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0dc1c6dbe5ca47f998dbd8ca60408c1c is 82, key is hbase:meta,,1/info:regioninfo/1733708728578/Put/seqid=0 2024-12-09T01:45:31,774 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0dc1c6dbe5ca47f998dbd8ca60408c1c 2024-12-09T01:45:31,782 DEBUG [M:0;ef6f18c58dc9:39747 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 29fe964f47f840be93155e4c624b2b59 2024-12-09T01:45:31,797 DEBUG [M:0;ef6f18c58dc9:39747 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/29fe964f47f840be93155e4c624b2b59 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733708729088/Put/seqid=0 2024-12-09T01:45:31,798 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/29fe964f47f840be93155e4c624b2b59 2024-12-09T01:45:31,807 DEBUG [M:0;ef6f18c58dc9:39747 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 89f4816124c3438d850a71262592679d 2024-12-09T01:45:31,822 DEBUG [M:0;ef6f18c58dc9:39747 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89f4816124c3438d850a71262592679d is 69, key is ef6f18c58dc9,36067,1733708730903/rs:state/1733708730942/Put/seqid=0 2024-12-09T01:45:31,823 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89f4816124c3438d850a71262592679d 2024-12-09T01:45:31,825 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 89f4816124c3438d850a71262592679d 2024-12-09T01:45:31,826 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/0dc1c6dbe5ca47f998dbd8ca60408c1c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0dc1c6dbe5ca47f998dbd8ca60408c1c 2024-12-09T01:45:31,828 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/0dc1c6dbe5ca47f998dbd8ca60408c1c, entries=8, sequenceid=83, filesize=5.5 K 2024-12-09T01:45:31,828 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/29fe964f47f840be93155e4c624b2b59 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/29fe964f47f840be93155e4c624b2b59 2024-12-09T01:45:31,830 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/29fe964f47f840be93155e4c624b2b59, entries=9, sequenceid=83, filesize=6.4 K 2024-12-09T01:45:31,830 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/89f4816124c3438d850a71262592679d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89f4816124c3438d850a71262592679d 2024-12-09T01:45:31,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:31,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42881-0x10074737ddf0002, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:31,831 INFO [RS:1;ef6f18c58dc9:42881 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:45:31,831 INFO [RS:1;ef6f18c58dc9:42881 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef6f18c58dc9,42881,1733708727656; zookeeper connection closed. 2024-12-09T01:45:31,832 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4ec99b1e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4ec99b1e 2024-12-09T01:45:31,832 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-09T01:45:31,832 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 89f4816124c3438d850a71262592679d 2024-12-09T01:45:31,833 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/5ebd1574-6062-5576-e299-626dcb1b95b0/f79f510f-5559-f342-538d-565c0cceb7ba/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/89f4816124c3438d850a71262592679d, entries=3, sequenceid=83, filesize=5.3 K 2024-12-09T01:45:31,833 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=83, compaction requested=false 2024-12-09T01:45:31,835 INFO [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:45:31,836 DEBUG [M:0;ef6f18c58dc9:39747 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733708731746Disabling compacts and flushes for region at 1733708731746Disabling writes for close at 1733708731746Obtaining lock to block concurrent updates at 1733708731747 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733708731747Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1733708731747Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733708731747Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733708731748 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733708731773 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733708731773Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733708731776 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733708731796 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733708731797 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733708731800 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733708731821 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733708731821Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c8cbf1d: reopening flushed file at 1733708731825 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c209ab9: reopening flushed file at 1733708731828 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@377f3466: reopening flushed file at 1733708731830 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=83, compaction requested=false at 1733708731833 (+3 ms)Writing region close event to WAL at 1733708731835 (+2 ms)Closed at 1733708731835 2024-12-09T01:45:31,837 INFO [M:0;ef6f18c58dc9:39747 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T01:45:31,837 INFO [M:0;ef6f18c58dc9:39747 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39747 2024-12-09T01:45:31,837 INFO [M:0;ef6f18c58dc9:39747 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:45:31,838 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:45:31,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:31,939 INFO [M:0;ef6f18c58dc9:39747 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:45:31,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39747-0x10074737ddf0000, quorum=127.0.0.1:53028, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:45:31,942 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T01:45:31,943 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-09T01:45:31,944 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:45:31,951 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 330) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11943 (was 12023)