2024-11-17 09:48:18,494 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-17 09:48:18,519 main DEBUG Took 0.012591 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-17 09:48:18,520 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-17 09:48:18,520 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-17 09:48:18,521 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-17 09:48:18,522 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,530 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-17 09:48:18,543 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,545 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,545 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,545 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,546 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,546 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,547 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,547 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,548 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,548 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,549 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,549 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,550 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,550 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,550 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,551 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,551 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,552 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,552 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,553 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,553 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,553 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,554 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-17 09:48:18,554 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,554 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-17 09:48:18,555 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-17 09:48:18,557 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-17 09:48:18,558 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-17 09:48:18,558 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-17 09:48:18,559 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-17 09:48:18,560 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-17 09:48:18,568 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-17 09:48:18,570 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-17 09:48:18,572 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-17 09:48:18,572 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-17 09:48:18,572 main DEBUG createAppenders(={Console}) 2024-11-17 09:48:18,573 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-17 09:48:18,573 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-17 09:48:18,573 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-17 09:48:18,574 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-17 09:48:18,574 main DEBUG OutputStream closed 2024-11-17 09:48:18,574 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-17 09:48:18,574 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-17 09:48:18,575 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-17 09:48:18,655 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-17 09:48:18,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-17 09:48:18,659 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-17 09:48:18,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-17 09:48:18,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-17 09:48:18,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-17 09:48:18,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-17 09:48:18,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-17 09:48:18,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-17 09:48:18,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-17 09:48:18,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-17 09:48:18,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-17 09:48:18,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-17 09:48:18,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-17 09:48:18,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-17 09:48:18,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-17 09:48:18,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-17 09:48:18,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-17 09:48:18,669 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17 09:48:18,670 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-17 09:48:18,670 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-17 09:48:18,671 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-17T09:48:18,687 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-17 09:48:18,690 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-17 09:48:18,690 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-17T09:48:18,992 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3 2024-11-17T09:48:19,021 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1273, ProcessCount=11, AvailableMemoryMB=3910 2024-11-17T09:48:19,142 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-17T09:48:19,245 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:19,261 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/cluster_0a174a88-75fb-34bb-21f6-4b3bbb238960, deleteOnExit=true 2024-11-17T09:48:19,351 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/cluster_0a174a88-75fb-34bb-21f6-4b3bbb238960/zookeeper_0, clientPort=64067, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/cluster_0a174a88-75fb-34bb-21f6-4b3bbb238960/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/cluster_0a174a88-75fb-34bb-21f6-4b3bbb238960/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T09:48:19,366 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64067 2024-11-17T09:48:19,366 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:19,460 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 with version=8 2024-11-17T09:48:19,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/hbase-staging 2024-11-17T09:48:19,562 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-17T09:48:19,840 INFO [Time-limited test {}] client.ConnectionUtils(128): master/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:48:19,851 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:19,852 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:19,860 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:48:19,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:19,860 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:48:19,998 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T09:48:20,084 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:48:20,121 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-17T09:48:20,132 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-17T09:48:20,136 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:48:20,174 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 21208 (auto-detected) 2024-11-17T09:48:20,175 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-17T09:48:20,195 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32885 2024-11-17T09:48:20,228 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:32885 connecting to ZooKeeper ensemble=127.0.0.1:64067 2024-11-17T09:48:20,265 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:328850x0, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:48:20,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:32885-0x1012553da900000 connected 2024-11-17T09:48:20,294 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:20,295 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:20,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:20,307 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8, hbase.cluster.distributed=false 2024-11-17T09:48:20,332 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:48:20,340 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32885 2024-11-17T09:48:20,341 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32885 2024-11-17T09:48:20,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32885 2024-11-17T09:48:20,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32885 2024-11-17T09:48:20,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32885 2024-11-17T09:48:20,434 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:48:20,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:20,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:20,437 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:48:20,437 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:20,437 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:48:20,440 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T09:48:20,442 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:48:20,443 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37353 2024-11-17T09:48:20,445 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37353 connecting to ZooKeeper ensemble=127.0.0.1:64067 2024-11-17T09:48:20,446 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:20,447 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:20,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:373530x0, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:48:20,451 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37353-0x1012553da900001 connected 2024-11-17T09:48:20,451 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:20,457 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T09:48:20,469 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.cache 2024-11-17T09:48:20,474 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T09:48:20,475 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T09:48:20,477 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T09:48:20,478 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T09:48:20,479 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T09:48:20,487 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T09:48:20,490 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T09:48:20,490 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.persistence is complete. 2024-11-17T09:48:20,494 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T09:48:20,497 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T09:48:20,503 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:48:20,504 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37353 2024-11-17T09:48:20,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37353 2024-11-17T09:48:20,505 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37353 2024-11-17T09:48:20,506 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37353 2024-11-17T09:48:20,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37353 2024-11-17T09:48:20,526 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:48:20,526 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:20,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:20,527 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:48:20,527 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:20,528 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:48:20,528 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T09:48:20,528 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:48:20,530 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36895 2024-11-17T09:48:20,531 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36895 connecting to ZooKeeper ensemble=127.0.0.1:64067 2024-11-17T09:48:20,532 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:20,533 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:20,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368950x0, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:48:20,538 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36895-0x1012553da900002 connected 2024-11-17T09:48:20,539 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:20,539 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T09:48:20,540 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.cache 2024-11-17T09:48:20,540 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T09:48:20,541 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T09:48:20,541 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T09:48:20,544 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T09:48:20,544 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T09:48:20,545 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T09:48:20,545 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T09:48:20,545 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T09:48:20,546 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.persistence is complete. 2024-11-17T09:48:20,546 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T09:48:20,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:48:20,552 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36895 2024-11-17T09:48:20,553 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36895 2024-11-17T09:48:20,556 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36895 2024-11-17T09:48:20,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36895 2024-11-17T09:48:20,557 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36895 2024-11-17T09:48:20,583 DEBUG [M:0;70e37c475432:32885 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;70e37c475432:32885 2024-11-17T09:48:20,589 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/70e37c475432,32885,1731836899642 2024-11-17T09:48:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:20,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:20,602 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/70e37c475432,32885,1731836899642 2024-11-17T09:48:20,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T09:48:20,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:20,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T09:48:20,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:20,634 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:20,635 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T09:48:20,637 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/70e37c475432,32885,1731836899642 from backup master directory 2024-11-17T09:48:20,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/70e37c475432,32885,1731836899642 2024-11-17T09:48:20,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:20,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:20,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:20,641 WARN [master/70e37c475432:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:48:20,642 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=70e37c475432,32885,1731836899642 2024-11-17T09:48:20,644 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-17T09:48:20,646 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-17T09:48:20,655 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/hbase.id] with ID: 722b84f3-b785-404e-82af-961d695d979e 2024-11-17T09:48:20,655 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.tmp/hbase.id 2024-11-17T09:48:20,662 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/hbase.id] 2024-11-17T09:48:20,707 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:20,714 INFO [master/70e37c475432:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T09:48:20,741 INFO [master/70e37c475432:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-17T09:48:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:20,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:20,776 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:48:20,778 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T09:48:20,831 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-17T09:48:20,838 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:20,903 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store 2024-11-17T09:48:20,929 INFO [master/70e37c475432:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-17T09:48:20,933 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:20,935 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T09:48:20,935 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:20,936 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:20,937 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T09:48:20,938 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:20,938 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:20,939 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731836900935Disabling compacts and flushes for region at 1731836900935Disabling writes for close at 1731836900937 (+2 ms)Writing region close event to WAL at 1731836900938 (+1 ms)Closed at 1731836900938 2024-11-17T09:48:20,939 WARN [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/.initializing 2024-11-17T09:48:20,939 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/WALs/70e37c475432,32885,1731836899642 2024-11-17T09:48:20,954 INFO [master/70e37c475432:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:48:20,967 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=70e37c475432%2C32885%2C1731836899642, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/WALs/70e37c475432,32885,1731836899642, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/oldWALs, maxLogs=10 2024-11-17T09:48:21,011 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/WALs/70e37c475432,32885,1731836899642/70e37c475432%2C32885%2C1731836899642.1731836900972 2024-11-17T09:48:21,012 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:21,013 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:21,014 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:21,018 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,020 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,073 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T09:48:21,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,120 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:21,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,127 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T09:48:21,127 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:21,128 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T09:48:21,134 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:21,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T09:48:21,140 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:21,142 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,145 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,146 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,149 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,150 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,154 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T09:48:21,156 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:21,174 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:48:21,175 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70716520, jitterRate=0.05375826358795166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T09:48:21,184 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731836901037Initializing all the Stores at 1731836901038 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836901039 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836901040 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836901040Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836901041 (+1 ms)Cleaning up temporary data from old regions at 1731836901150 (+109 ms)Region opened successfully at 1731836901184 (+34 ms) 2024-11-17T09:48:21,186 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T09:48:21,228 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226a2b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:48:21,273 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T09:48:21,287 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T09:48:21,288 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T09:48:21,291 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T09:48:21,291 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T09:48:21,299 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-17T09:48:21,299 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T09:48:21,325 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T09:48:21,336 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T09:48:21,340 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T09:48:21,342 INFO [master/70e37c475432:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T09:48:21,344 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T09:48:21,346 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T09:48:21,348 INFO [master/70e37c475432:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T09:48:21,351 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T09:48:21,353 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T09:48:21,354 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T09:48:21,356 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T09:48:21,375 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T09:48:21,377 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T09:48:21,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:21,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:21,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:21,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,396 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=70e37c475432,32885,1731836899642, sessionid=0x1012553da900000, setting cluster-up flag (Was=false) 2024-11-17T09:48:21,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,410 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,419 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T09:48:21,421 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=70e37c475432,32885,1731836899642 2024-11-17T09:48:21,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:21,431 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T09:48:21,434 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=70e37c475432,32885,1731836899642 2024-11-17T09:48:21,440 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T09:48:21,479 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(746): ClusterId : 722b84f3-b785-404e-82af-961d695d979e 2024-11-17T09:48:21,479 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(746): ClusterId : 722b84f3-b785-404e-82af-961d695d979e 2024-11-17T09:48:21,483 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T09:48:21,483 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T09:48:21,489 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T09:48:21,489 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T09:48:21,489 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T09:48:21,489 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T09:48:21,494 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T09:48:21,494 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T09:48:21,494 DEBUG [RS:0;70e37c475432:37353 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ddc404f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:48:21,495 DEBUG [RS:1;70e37c475432:36895 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ebd5e7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:48:21,515 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;70e37c475432:37353 2024-11-17T09:48:21,519 INFO [RS:0;70e37c475432:37353 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T09:48:21,519 INFO [RS:0;70e37c475432:37353 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T09:48:21,519 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T09:48:21,520 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;70e37c475432:36895 2024-11-17T09:48:21,520 INFO [RS:1;70e37c475432:36895 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T09:48:21,521 INFO [RS:1;70e37c475432:36895 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T09:48:21,521 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T09:48:21,522 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,32885,1731836899642 with port=36895, startcode=1731836900525 2024-11-17T09:48:21,522 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,32885,1731836899642 with port=37353, startcode=1731836900395 2024-11-17T09:48:21,536 DEBUG [RS:1;70e37c475432:36895 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T09:48:21,536 DEBUG [RS:0;70e37c475432:37353 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T09:48:21,536 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:21,551 INFO [master/70e37c475432:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T09:48:21,560 INFO [master/70e37c475432:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T09:48:21,566 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 70e37c475432,32885,1731836899642 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T09:48:21,574 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:21,575 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:21,575 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:21,575 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:21,576 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/70e37c475432:0, corePoolSize=10, maxPoolSize=10 2024-11-17T09:48:21,576 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,576 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:48:21,577 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,590 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:21,590 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T09:48:21,595 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,595 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T09:48:21,618 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731836931618 2024-11-17T09:48:21,619 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54843, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T09:48:21,619 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52991, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T09:48:21,619 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T09:48:21,621 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T09:48:21,624 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T09:48:21,625 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T09:48:21,625 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T09:48:21,625 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T09:48:21,627 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:48:21,631 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,633 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-17T09:48:21,634 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T09:48:21,637 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T09:48:21,637 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:21,638 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T09:48:21,638 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T09:48:21,643 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T09:48:21,644 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T09:48:21,649 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836901645,5,FailOnTimeoutGroup] 2024-11-17T09:48:21,652 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836901650,5,FailOnTimeoutGroup] 2024-11-17T09:48:21,652 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,652 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T09:48:21,654 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,654 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,658 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:21,659 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T09:48:21,664 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T09:48:21,664 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-17T09:48:21,664 WARN [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T09:48:21,664 WARN [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-17T09:48:21,665 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T09:48:21,666 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:21,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T09:48:21,673 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T09:48:21,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,673 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:21,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T09:48:21,680 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T09:48:21,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,680 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:21,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T09:48:21,687 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T09:48:21,687 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:21,688 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:21,688 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T09:48:21,689 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:21,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:21,690 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T09:48:21,691 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T09:48:21,691 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T09:48:21,692 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T09:48:21,700 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:48:21,700 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72402860, jitterRate=0.07888668775558472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T09:48:21,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731836901658Initializing all the Stores at 1731836901658Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836901658Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836901659 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836901659Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836901659Cleaning up temporary data from old regions at 1731836901691 (+32 ms)Region opened successfully at 1731836901703 (+12 ms) 2024-11-17T09:48:21,703 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T09:48:21,704 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T09:48:21,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T09:48:21,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T09:48:21,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T09:48:21,705 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T09:48:21,706 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731836901703Disabling compacts and flushes for region at 1731836901703Disabling writes for close at 1731836901704 (+1 ms)Writing region close event to WAL at 1731836901705 (+1 ms)Closed at 1731836901705 2024-11-17T09:48:21,708 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:21,708 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T09:48:21,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T09:48:21,723 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T09:48:21,726 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T09:48:21,766 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,32885,1731836899642 with port=36895, startcode=1731836900525 2024-11-17T09:48:21,766 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,32885,1731836899642 with port=37353, startcode=1731836900395 2024-11-17T09:48:21,768 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 70e37c475432,37353,1731836900395 2024-11-17T09:48:21,772 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(517): Registering regionserver=70e37c475432,37353,1731836900395 2024-11-17T09:48:21,781 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 70e37c475432,36895,1731836900525 2024-11-17T09:48:21,781 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=32885 {}] master.ServerManager(517): Registering regionserver=70e37c475432,36895,1731836900525 2024-11-17T09:48:21,782 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:21,782 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T09:48:21,782 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T09:48:21,784 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:21,784 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T09:48:21,784 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T09:48:21,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:48:21,792 DEBUG [RS:0;70e37c475432:37353 {}] zookeeper.ZKUtil(111): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/70e37c475432,37353,1731836900395 2024-11-17T09:48:21,792 WARN [RS:0;70e37c475432:37353 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:48:21,793 INFO [RS:0;70e37c475432:37353 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:21,793 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,37353,1731836900395 2024-11-17T09:48:21,794 DEBUG [RS:1;70e37c475432:36895 {}] zookeeper.ZKUtil(111): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/70e37c475432,36895,1731836900525 2024-11-17T09:48:21,794 WARN [RS:1;70e37c475432:36895 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:48:21,794 INFO [RS:1;70e37c475432:36895 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:21,794 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,36895,1731836900525 2024-11-17T09:48:21,795 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [70e37c475432,36895,1731836900525] 2024-11-17T09:48:21,795 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [70e37c475432,37353,1731836900395] 2024-11-17T09:48:21,824 INFO [RS:0;70e37c475432:37353 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T09:48:21,824 INFO [RS:1;70e37c475432:36895 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T09:48:21,840 INFO [RS:1;70e37c475432:36895 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T09:48:21,840 INFO [RS:0;70e37c475432:37353 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T09:48:21,845 INFO [RS:1;70e37c475432:36895 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T09:48:21,845 INFO [RS:0;70e37c475432:37353 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T09:48:21,846 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,846 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,847 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T09:48:21,847 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T09:48:21,853 INFO [RS:1;70e37c475432:36895 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T09:48:21,853 INFO [RS:0;70e37c475432:37353 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T09:48:21,855 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,855 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,856 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,856 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,856 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,856 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,856 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,856 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,856 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,856 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,857 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,857 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,857 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:48:21,857 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:48:21,857 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,857 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,857 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,857 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,858 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:21,859 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:21,859 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:21,859 DEBUG [RS:0;70e37c475432:37353 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:21,859 DEBUG [RS:1;70e37c475432:36895 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:21,864 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,865 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,865 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,865 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,865 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,865 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,865 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,36895,1731836900525-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:48:21,866 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,866 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,867 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,867 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,867 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,37353,1731836900395-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:48:21,877 WARN [70e37c475432:32885 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T09:48:21,888 INFO [RS:1;70e37c475432:36895 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T09:48:21,890 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,36895,1731836900525-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,890 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,890 INFO [RS:1;70e37c475432:36895 {}] regionserver.Replication(171): 70e37c475432,36895,1731836900525 started 2024-11-17T09:48:21,893 INFO [RS:0;70e37c475432:37353 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T09:48:21,893 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,37353,1731836900395-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,894 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,894 INFO [RS:0;70e37c475432:37353 {}] regionserver.Replication(171): 70e37c475432,37353,1731836900395 started 2024-11-17T09:48:21,910 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,910 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1482): Serving as 70e37c475432,36895,1731836900525, RpcServer on 70e37c475432/172.17.0.2:36895, sessionid=0x1012553da900002 2024-11-17T09:48:21,911 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T09:48:21,911 DEBUG [RS:1;70e37c475432:36895 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 70e37c475432,36895,1731836900525 2024-11-17T09:48:21,912 DEBUG [RS:1;70e37c475432:36895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,36895,1731836900525' 2024-11-17T09:48:21,912 DEBUG [RS:1;70e37c475432:36895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T09:48:21,913 DEBUG [RS:1;70e37c475432:36895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T09:48:21,915 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T09:48:21,915 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T09:48:21,915 DEBUG [RS:1;70e37c475432:36895 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 70e37c475432,36895,1731836900525 2024-11-17T09:48:21,915 DEBUG [RS:1;70e37c475432:36895 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,36895,1731836900525' 2024-11-17T09:48:21,916 DEBUG [RS:1;70e37c475432:36895 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T09:48:21,917 DEBUG [RS:1;70e37c475432:36895 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T09:48:21,917 DEBUG [RS:1;70e37c475432:36895 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T09:48:21,917 INFO [RS:1;70e37c475432:36895 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T09:48:21,918 INFO [RS:1;70e37c475432:36895 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T09:48:21,918 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:21,918 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1482): Serving as 70e37c475432,37353,1731836900395, RpcServer on 70e37c475432/172.17.0.2:37353, sessionid=0x1012553da900001 2024-11-17T09:48:21,918 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T09:48:21,918 DEBUG [RS:0;70e37c475432:37353 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 70e37c475432,37353,1731836900395 2024-11-17T09:48:21,918 DEBUG [RS:0;70e37c475432:37353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,37353,1731836900395' 2024-11-17T09:48:21,919 DEBUG [RS:0;70e37c475432:37353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T09:48:21,919 DEBUG [RS:0;70e37c475432:37353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T09:48:21,920 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T09:48:21,920 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T09:48:21,921 DEBUG [RS:0;70e37c475432:37353 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 70e37c475432,37353,1731836900395 2024-11-17T09:48:21,921 DEBUG [RS:0;70e37c475432:37353 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,37353,1731836900395' 2024-11-17T09:48:21,921 DEBUG [RS:0;70e37c475432:37353 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T09:48:21,922 DEBUG [RS:0;70e37c475432:37353 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T09:48:21,922 DEBUG [RS:0;70e37c475432:37353 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T09:48:21,923 INFO [RS:0;70e37c475432:37353 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T09:48:21,923 INFO [RS:0;70e37c475432:37353 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T09:48:22,023 INFO [RS:1;70e37c475432:36895 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:48:22,024 INFO [RS:0;70e37c475432:37353 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:48:22,025 INFO [RS:0;70e37c475432:37353 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=70e37c475432%2C37353%2C1731836900395, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,37353,1731836900395, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs, maxLogs=55 2024-11-17T09:48:22,025 INFO [RS:1;70e37c475432:36895 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=70e37c475432%2C36895%2C1731836900525, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,36895,1731836900525, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs, maxLogs=55 2024-11-17T09:48:22,049 INFO [RS:0;70e37c475432:37353 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,37353,1731836900395/70e37c475432%2C37353%2C1731836900395.1731836902029 2024-11-17T09:48:22,049 INFO [RS:1;70e37c475432:36895 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,36895,1731836900525/70e37c475432%2C36895%2C1731836900525.1731836902030 2024-11-17T09:48:22,049 DEBUG [RS:0;70e37c475432:37353 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:22,050 DEBUG [RS:1;70e37c475432:36895 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:22,129 DEBUG [70e37c475432:32885 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-17T09:48:22,145 DEBUG [70e37c475432:32885 {}] balancer.BalancerClusterState(204): Hosts are {70e37c475432=0} racks are {/default-rack=0} 2024-11-17T09:48:22,149 DEBUG [70e37c475432:32885 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T09:48:22,149 DEBUG [70e37c475432:32885 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T09:48:22,149 DEBUG [70e37c475432:32885 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T09:48:22,149 DEBUG [70e37c475432:32885 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T09:48:22,149 INFO [70e37c475432:32885 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T09:48:22,149 INFO [70e37c475432:32885 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T09:48:22,149 DEBUG [70e37c475432:32885 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T09:48:22,157 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=70e37c475432,37353,1731836900395 2024-11-17T09:48:22,163 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 70e37c475432,37353,1731836900395, state=OPENING 2024-11-17T09:48:22,169 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T09:48:22,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:22,175 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:22,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:22,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:22,177 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:22,179 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:22,180 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T09:48:22,182 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=70e37c475432,37353,1731836900395}] 2024-11-17T09:48:22,358 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T09:48:22,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47147, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T09:48:22,369 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T09:48:22,369 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:22,370 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T09:48:22,371 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=70e37c475432%2C37353%2C1731836900395.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,37353,1731836900395, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs, maxLogs=55 2024-11-17T09:48:22,394 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,37353,1731836900395/70e37c475432%2C37353%2C1731836900395.meta.1731836902373.meta 2024-11-17T09:48:22,395 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:22,395 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:22,397 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T09:48:22,400 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T09:48:22,406 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T09:48:22,410 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T09:48:22,410 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:22,411 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T09:48:22,411 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T09:48:22,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T09:48:22,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T09:48:22,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:22,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:22,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T09:48:22,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T09:48:22,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:22,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:22,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T09:48:22,416 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T09:48:22,416 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:22,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:22,417 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T09:48:22,417 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T09:48:22,418 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:22,418 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:22,418 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T09:48:22,419 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:22,420 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:22,420 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T09:48:22,421 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T09:48:22,421 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T09:48:22,422 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T09:48:22,423 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65111571, jitterRate=-0.029761984944343567}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T09:48:22,423 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T09:48:22,424 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731836902411Writing region info on filesystem at 1731836902412 (+1 ms)Initializing all the Stores at 1731836902412Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836902412Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836902412Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836902412Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836902412Cleaning up temporary data from old regions at 1731836902421 (+9 ms)Running coprocessor post-open hooks at 1731836902423 (+2 ms)Region opened successfully at 1731836902424 (+1 ms) 2024-11-17T09:48:22,430 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731836902347 2024-11-17T09:48:22,441 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T09:48:22,442 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T09:48:22,443 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,37353,1731836900395 2024-11-17T09:48:22,444 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 70e37c475432,37353,1731836900395, state=OPEN 2024-11-17T09:48:22,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:48:22,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:48:22,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:48:22,446 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:22,446 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:22,447 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:22,447 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=70e37c475432,37353,1731836900395 2024-11-17T09:48:22,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T09:48:22,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=70e37c475432,37353,1731836900395 in 265 msec 2024-11-17T09:48:22,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T09:48:22,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 736 msec 2024-11-17T09:48:22,455 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:22,455 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T09:48:22,476 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:48:22,477 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,37353,1731836900395, seqNum=-1] 2024-11-17T09:48:22,502 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:48:22,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50563, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:48:22,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0420 sec 2024-11-17T09:48:22,523 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731836902522, completionTime=-1 2024-11-17T09:48:22,525 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-17T09:48:22,525 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T09:48:22,551 INFO [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T09:48:22,552 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731836962551 2024-11-17T09:48:22,552 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731837022552 2024-11-17T09:48:22,552 INFO [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-17T09:48:22,554 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-17T09:48:22,560 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,32885,1731836899642-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:22,560 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,32885,1731836899642-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:22,560 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,32885,1731836899642-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:22,562 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-70e37c475432:32885, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:22,562 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:22,563 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:22,569 DEBUG [master/70e37c475432:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T09:48:22,596 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.954sec 2024-11-17T09:48:22,598 INFO [master/70e37c475432:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T09:48:22,599 INFO [master/70e37c475432:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T09:48:22,600 INFO [master/70e37c475432:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T09:48:22,600 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T09:48:22,601 INFO [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T09:48:22,601 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,32885,1731836899642-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:48:22,602 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,32885,1731836899642-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T09:48:22,603 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T09:48:22,604 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T09:48:22,604 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,32885,1731836899642-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:22,710 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d0b20dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:48:22,715 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-17T09:48:22,716 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-17T09:48:22,721 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,32885,-1 for getting cluster id 2024-11-17T09:48:22,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:48:22,731 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '722b84f3-b785-404e-82af-961d695d979e' 2024-11-17T09:48:22,733 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:48:22,733 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "722b84f3-b785-404e-82af-961d695d979e" 2024-11-17T09:48:22,734 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a490df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:48:22,734 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,32885,-1] 2024-11-17T09:48:22,736 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:48:22,738 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:22,739 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:48:22,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:48:22,742 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:48:22,749 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,37353,1731836900395, seqNum=-1] 2024-11-17T09:48:22,749 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:48:22,752 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:48:22,770 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=70e37c475432,32885,1731836899642 2024-11-17T09:48:22,774 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:48:22,778 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,32885,1731836899642 2024-11-17T09:48:22,781 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2a9c1634 2024-11-17T09:48:22,782 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:48:22,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35280, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:48:22,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731836902770', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:48:22,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 2024-11-17T09:48:22,799 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:48:22,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731836902770" procId is: 4 2024-11-17T09:48:22,801 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:22,803 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:48:22,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:22,825 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab008830583c69e6dce70419d27e8e01, NAME => 'testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731836902770', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:22,832 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:22,832 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(1722): Closing ab008830583c69e6dce70419d27e8e01, disabling compactions & flushes 2024-11-17T09:48:22,832 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:22,832 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:22,832 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. after waiting 0 ms 2024-11-17T09:48:22,833 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:22,833 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:22,833 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731836902770-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab008830583c69e6dce70419d27e8e01: Waiting for close lock at 1731836902832Disabling compacts and flushes for region at 1731836902832Disabling writes for close at 1731836902832Writing region close event to WAL at 1731836902833 (+1 ms)Closed at 1731836902833 2024-11-17T09:48:22,834 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:48:22,839 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731836902834"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836902834"}]},"ts":"1731836902834"} 2024-11-17T09:48:22,844 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:48:22,845 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:48:22,847 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731836902770","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836902845"}]},"ts":"1731836902845"} 2024-11-17T09:48:22,851 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731836902770, state=ENABLING in hbase:meta 2024-11-17T09:48:22,852 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {70e37c475432=0} racks are {/default-rack=0} 2024-11-17T09:48:22,852 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T09:48:22,852 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T09:48:22,852 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T09:48:22,852 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T09:48:22,852 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T09:48:22,852 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T09:48:22,852 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T09:48:22,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, ASSIGN}] 2024-11-17T09:48:22,855 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, ASSIGN 2024-11-17T09:48:22,856 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, ASSIGN; state=OFFLINE, location=70e37c475432,37353,1731836900395; forceNewPlan=false, retain=false 2024-11-17T09:48:22,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:23,009 INFO [70e37c475432:32885 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T09:48:23,009 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ab008830583c69e6dce70419d27e8e01, regionState=OPENING, regionLocation=70e37c475432,37353,1731836900395 2024-11-17T09:48:23,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, ASSIGN because future has completed 2024-11-17T09:48:23,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,37353,1731836900395}] 2024-11-17T09:48:23,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:23,170 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:23,170 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ab008830583c69e6dce70419d27e8e01, NAME => 'testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:23,171 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731836902770 ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,171 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:23,171 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,171 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,172 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,177 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab008830583c69e6dce70419d27e8e01 columnFamilyName family 2024-11-17T09:48:23,177 DEBUG [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:23,178 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.HStore(327): Store=ab008830583c69e6dce70419d27e8e01/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:23,178 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,179 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,179 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,179 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,179 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,180 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,187 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:48:23,188 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ab008830583c69e6dce70419d27e8e01; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65314784, jitterRate=-0.026733875274658203}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T09:48:23,188 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:23,189 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ab008830583c69e6dce70419d27e8e01: Running coprocessor pre-open hook at 1731836903171Writing region info on filesystem at 1731836903171Initializing all the Stores at 1731836903172 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836903172Cleaning up temporary data from old regions at 1731836903179 (+7 ms)Running coprocessor post-open hooks at 1731836903188 (+9 ms)Region opened successfully at 1731836903189 (+1 ms) 2024-11-17T09:48:23,190 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01., pid=6, masterSystemTime=1731836903167 2024-11-17T09:48:23,193 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:23,193 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:23,194 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ab008830583c69e6dce70419d27e8e01, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,37353,1731836900395 2024-11-17T09:48:23,197 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,37353,1731836900395 because future has completed 2024-11-17T09:48:23,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T09:48:23,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,37353,1731836900395 in 184 msec 2024-11-17T09:48:23,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T09:48:23,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, ASSIGN in 346 msec 2024-11-17T09:48:23,203 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:48:23,203 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731836902770","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836903203"}]},"ts":"1731836903203"} 2024-11-17T09:48:23,206 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731836902770, state=ENABLED in hbase:meta 2024-11-17T09:48:23,207 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:48:23,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731836902770 in 414 msec 2024-11-17T09:48:23,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:23,437 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731836902770 completed 2024-11-17T09:48:23,437 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731836902770 get assigned. Timeout = 60000ms 2024-11-17T09:48:23,440 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T09:48:23,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731836902770 assigned to meta. Checking AM states. 2024-11-17T09:48:23,445 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T09:48:23,446 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731836902770 assigned. 2024-11-17T09:48:23,453 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731836902770', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01., hostname=70e37c475432,37353,1731836900395, seqNum=2] 2024-11-17T09:48:23,463 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ab008830583c69e6dce70419d27e8e01 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-17T09:48:23,489 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:23,525 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/.tmp/family/4b1205b68b35475db2e32c28e3dfc2ba is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-17T09:48:23,530 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/.tmp/family/4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:23,565 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/.tmp/family/4b1205b68b35475db2e32c28e3dfc2ba as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:23,567 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba, delay=950 ms 2024-11-17T09:48:23,568 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba, entries=2, sequenceid=6, filesize=4.9 K 2024-11-17T09:48:23,574 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ab008830583c69e6dce70419d27e8e01 in 110ms, sequenceid=6, compaction requested=false 2024-11-17T09:48:23,575 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731836902770' 2024-11-17T09:48:23,576 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ab008830583c69e6dce70419d27e8e01: 2024-11-17T09:48:24,519 DEBUG [hfile-prefetch-1731836903567 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4b1205b68b35475db2e32c28e3dfc2ba_0. 2024-11-17T09:48:24,523 DEBUG [hfile-prefetch-1731836903567 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4b1205b68b35475db2e32c28e3dfc2ba_117. 2024-11-17T09:48:24,524 DEBUG [hfile-prefetch-1731836903567 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba, with total blocks 2, and data blocks 1 2024-11-17T09:48:24,524 DEBUG [hfile-prefetch-1731836903567 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-17T09:48:24,525 DEBUG [hfile-prefetch-1731836903567 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-17T09:48:24,525 DEBUG [hfile-prefetch-1731836903567 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba. Will sleep 100ms and try the verification again. 2024-11-17T09:48:24,544 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T09:48:24,544 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T09:48:24,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:24,545 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T09:48:24,562 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T09:48:24,562 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731836904544 2024-11-17T09:48:24,562 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T09:48:24,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=ab008830583c69e6dce70419d27e8e01, source=70e37c475432,37353,1731836900395, destination=70e37c475432,36895,1731836900525, warming up region on 70e37c475432,36895,1731836900525 2024-11-17T09:48:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T09:48:24,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=ab008830583c69e6dce70419d27e8e01, source=70e37c475432,37353,1731836900395, destination=70e37c475432,36895,1731836900525, running balancer 2024-11-17T09:48:24,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39915, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T09:48:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, REOPEN/MOVE 2024-11-17T09:48:24,592 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, REOPEN/MOVE 2024-11-17T09:48:24,593 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(7855): Warmup {ENCODED => ab008830583c69e6dce70419d27e8e01, NAME => 'testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:24,594 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab008830583c69e6dce70419d27e8e01, regionState=CLOSING, regionLocation=70e37c475432,37353,1731836900395 2024-11-17T09:48:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:24,595 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:24,596 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab008830583c69e6dce70419d27e8e01 columnFamilyName family 2024-11-17T09:48:24,596 DEBUG [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:24,597 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, REOPEN/MOVE because future has completed 2024-11-17T09:48:24,597 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-17T09:48:24,597 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,37353,1731836900395}] 2024-11-17T09:48:24,600 DEBUG [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:24,600 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.HStore(327): Store=ab008830583c69e6dce70419d27e8e01/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(1722): Closing ab008830583c69e6dce70419d27e8e01, disabling compactions & flushes 2024-11-17T09:48:24,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. after waiting 0 ms 2024-11-17T09:48:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,602 DEBUG [StoreFileCloser-ab008830583c69e6dce70419d27e8e01-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:24,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36895 {}] regionserver.HRegion(1676): Region close journal for ab008830583c69e6dce70419d27e8e01: Waiting for close lock at 1731836904601Disabling compacts and flushes for region at 1731836904601Disabling writes for close at 1731836904601Writing region close event to WAL at 1731836904603 (+2 ms)Closed at 1731836904603 2024-11-17T09:48:24,604 WARN [hfile-prefetch-1731836903567 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-17T09:48:24,605 DEBUG [hfile-prefetch-1731836903567 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=32885 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-17T09:48:24,756 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:24,756 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-17T09:48:24,757 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing ab008830583c69e6dce70419d27e8e01, disabling compactions & flushes 2024-11-17T09:48:24,757 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,757 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,757 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. after waiting 0 ms 2024-11-17T09:48:24,757 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,759 DEBUG [StoreFileCloser-ab008830583c69e6dce70419d27e8e01-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 4b1205b68b35475db2e32c28e3dfc2ba, starting offset: 0, end offset: 9223372036854775807 2024-11-17T09:48:24,760 DEBUG [StoreFileCloser-ab008830583c69e6dce70419d27e8e01-family-1 {}] bucket.BucketCache(856): removed key 4b1205b68b35475db2e32c28e3dfc2ba_0 from back map with offset lock 2094080 in the evict process 2024-11-17T09:48:24,762 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T09:48:24,762 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T09:48:24,762 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:24,762 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T09:48:24,766 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T09:48:24,766 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731836904762 2024-11-17T09:48:24,766 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T09:48:24,766 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-17T09:48:24,768 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:24,768 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for ab008830583c69e6dce70419d27e8e01: Waiting for close lock at 1731836904757Running coprocessor pre-close hooks at 1731836904757Disabling compacts and flushes for region at 1731836904757Disabling writes for close at 1731836904757Writing region close event to WAL at 1731836904761 (+4 ms)Running coprocessor post-close hooks at 1731836904766 (+5 ms)Closed at 1731836904767 (+1 ms) 2024-11-17T09:48:24,768 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding ab008830583c69e6dce70419d27e8e01 move to 70e37c475432,36895,1731836900525 record at close sequenceid=6 2024-11-17T09:48:24,771 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:24,772 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab008830583c69e6dce70419d27e8e01, regionState=CLOSED 2024-11-17T09:48:24,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,37353,1731836900395 because future has completed 2024-11-17T09:48:24,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T09:48:24,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,37353,1731836900395 in 178 msec 2024-11-17T09:48:24,778 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, REOPEN/MOVE; state=CLOSED, location=70e37c475432,36895,1731836900525; forceNewPlan=false, retain=false 2024-11-17T09:48:24,929 INFO [70e37c475432:32885 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T09:48:24,929 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab008830583c69e6dce70419d27e8e01, regionState=OPENING, regionLocation=70e37c475432,36895,1731836900525 2024-11-17T09:48:24,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, REOPEN/MOVE because future has completed 2024-11-17T09:48:24,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,36895,1731836900525}] 2024-11-17T09:48:25,087 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,088 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ab008830583c69e6dce70419d27e8e01, NAME => 'testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:25,088 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731836902770 ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,088 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:25,089 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,089 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,090 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,091 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab008830583c69e6dce70419d27e8e01 columnFamilyName family 2024-11-17T09:48:25,091 DEBUG [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:25,093 DEBUG [StoreFileOpener-ab008830583c69e6dce70419d27e8e01-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba, delay=941 ms 2024-11-17T09:48:25,094 DEBUG [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:25,094 INFO [StoreOpener-ab008830583c69e6dce70419d27e8e01-1 {}] regionserver.HStore(327): Store=ab008830583c69e6dce70419d27e8e01/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:25,094 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,095 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,096 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,096 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,096 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,102 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,103 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ab008830583c69e6dce70419d27e8e01; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71718844, jitterRate=0.06869405508041382}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T09:48:25,103 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,104 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ab008830583c69e6dce70419d27e8e01: Running coprocessor pre-open hook at 1731836905089Writing region info on filesystem at 1731836905089Initializing all the Stores at 1731836905089Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836905089Cleaning up temporary data from old regions at 1731836905096 (+7 ms)Running coprocessor post-open hooks at 1731836905103 (+7 ms)Region opened successfully at 1731836905104 (+1 ms) 2024-11-17T09:48:25,105 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01., pid=9, masterSystemTime=1731836905085 2024-11-17T09:48:25,108 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,108 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,108 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ab008830583c69e6dce70419d27e8e01, regionState=OPEN, openSeqNum=10, regionLocation=70e37c475432,36895,1731836900525 2024-11-17T09:48:25,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,36895,1731836900525 because future has completed 2024-11-17T09:48:25,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-17T09:48:25,114 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ab008830583c69e6dce70419d27e8e01, server=70e37c475432,36895,1731836900525 in 179 msec 2024-11-17T09:48:25,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731836902770, region=ab008830583c69e6dce70419d27e8e01, REOPEN/MOVE in 524 msec 2024-11-17T09:48:25,144 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T09:48:25,144 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:48:25,144 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:25,149 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:25,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:25,150 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:48:25,150 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T09:48:25,150 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-17T09:48:25,150 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=70e37c475432,32885,1731836899642 2024-11-17T09:48:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T09:48:25,153 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:48:25,153 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:25,153 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:25,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:25,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:25,154 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '70e37c475432,37353,1731836900395' ***** 2024-11-17T09:48:25,154 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:25,154 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T09:48:25,154 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '70e37c475432,36895,1731836900525' ***** 2024-11-17T09:48:25,154 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T09:48:25,154 INFO [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T09:48:25,154 INFO [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T09:48:25,155 INFO [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.persistence 2024-11-17T09:48:25,155 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T09:48:25,155 INFO [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/bucket.persistence 2024-11-17T09:48:25,155 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T09:48:25,155 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T09:48:25,155 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T09:48:25,155 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T09:48:25,155 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T09:48:25,155 INFO [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T09:48:25,155 INFO [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T09:48:25,155 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T09:48:25,155 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T09:48:25,164 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T09:48:25,164 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T09:48:25,165 DEBUG [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1487): Thread RS:0;70e37c475432:37353 started persisting bucket cache to file 2024-11-17T09:48:25,165 DEBUG [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1487): Thread RS:1;70e37c475432:36895 started persisting bucket cache to file 2024-11-17T09:48:25,165 DEBUG [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:25,165 DEBUG [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:25,165 DEBUG [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T09:48:25,165 DEBUG [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T09:48:25,169 DEBUG [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T09:48:25,169 DEBUG [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731836905165 2024-11-17T09:48:25,169 DEBUG [RS:1;70e37c475432:36895 {}] bucket.BucketCache(1509): Thread RS:1;70e37c475432:36895 finished persisting bucket cache to file, renaming 2024-11-17T09:48:25,169 DEBUG [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T09:48:25,169 INFO [RS:1;70e37c475432:36895 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T09:48:25,169 DEBUG [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731836905165 2024-11-17T09:48:25,169 DEBUG [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1509): Thread RS:0;70e37c475432:37353 finished persisting bucket cache to file, renaming 2024-11-17T09:48:25,169 WARN [RS:0;70e37c475432:37353 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-17T09:48:25,170 INFO [RS:1;70e37c475432:36895 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T09:48:25,170 INFO [regionserver/70e37c475432:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:25,170 INFO [RS:1;70e37c475432:36895 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T09:48:25,170 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(3091): Received CLOSE for ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(959): stopping server 70e37c475432,37353,1731836900395 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:48:25,170 INFO [regionserver/70e37c475432:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;70e37c475432:37353. 2024-11-17T09:48:25,170 DEBUG [RS:0;70e37c475432:37353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:25,170 DEBUG [RS:0;70e37c475432:37353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:25,170 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(959): stopping server 70e37c475432,36895,1731836900525 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T09:48:25,170 INFO [RS:1;70e37c475432:36895 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:48:25,170 INFO [RS:0;70e37c475432:37353 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T09:48:25,171 INFO [RS:0;70e37c475432:37353 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T09:48:25,171 INFO [RS:1;70e37c475432:36895 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;70e37c475432:36895. 2024-11-17T09:48:25,171 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T09:48:25,171 DEBUG [RS:1;70e37c475432:36895 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:25,171 DEBUG [RS:1;70e37c475432:36895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:25,171 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T09:48:25,171 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T09:48:25,171 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T09:48:25,171 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-17T09:48:25,171 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1325): Online Regions={ab008830583c69e6dce70419d27e8e01=testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01.} 2024-11-17T09:48:25,171 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ab008830583c69e6dce70419d27e8e01, disabling compactions & flushes 2024-11-17T09:48:25,171 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,171 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,171 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. after waiting 0 ms 2024-11-17T09:48:25,171 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,171 DEBUG [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1351): Waiting on ab008830583c69e6dce70419d27e8e01 2024-11-17T09:48:25,171 DEBUG [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-17T09:48:25,171 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T09:48:25,172 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T09:48:25,172 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T09:48:25,172 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T09:48:25,172 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T09:48:25,172 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T09:48:25,172 DEBUG [StoreFileCloser-ab008830583c69e6dce70419d27e8e01-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/family/4b1205b68b35475db2e32c28e3dfc2ba 2024-11-17T09:48:25,172 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-17T09:48:25,177 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnRegionMove1731836902770/ab008830583c69e6dce70419d27e8e01/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-17T09:48:25,177 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,177 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ab008830583c69e6dce70419d27e8e01: Waiting for close lock at 1731836905171Running coprocessor pre-close hooks at 1731836905171Disabling compacts and flushes for region at 1731836905171Disabling writes for close at 1731836905171Writing region close event to WAL at 1731836905172 (+1 ms)Running coprocessor post-close hooks at 1731836905177 (+5 ms)Closed at 1731836905177 2024-11-17T09:48:25,177 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01. 2024-11-17T09:48:25,184 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7ec862786f1b451c9ef328796db5f0be 2024-11-17T09:48:25,203 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/info/7ec862786f1b451c9ef328796db5f0be is 205, key is testBlockEvictionOnRegionMove1731836902770,,1731836902786.ab008830583c69e6dce70419d27e8e01./info:regioninfo/1731836905108/Put/seqid=0 2024-11-17T09:48:25,204 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/info/7ec862786f1b451c9ef328796db5f0be 2024-11-17T09:48:25,219 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2923e082527f492586ace479ab7f29dd 2024-11-17T09:48:25,235 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/ns/2923e082527f492586ace479ab7f29dd is 43, key is default/ns:d/1731836902509/Put/seqid=0 2024-11-17T09:48:25,236 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/ns/2923e082527f492586ace479ab7f29dd 2024-11-17T09:48:25,247 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ba361d0eeb724cf2891dd209247ee746 2024-11-17T09:48:25,263 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/table/ba361d0eeb724cf2891dd209247ee746 is 78, key is testBlockEvictionOnRegionMove1731836902770/table:state/1731836903203/Put/seqid=0 2024-11-17T09:48:25,264 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/table/ba361d0eeb724cf2891dd209247ee746 2024-11-17T09:48:25,267 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/info/7ec862786f1b451c9ef328796db5f0be as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/7ec862786f1b451c9ef328796db5f0be 2024-11-17T09:48:25,269 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/7ec862786f1b451c9ef328796db5f0be, delay=915 ms 2024-11-17T09:48:25,271 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/7ec862786f1b451c9ef328796db5f0be, entries=15, sequenceid=15, filesize=7.6 K 2024-11-17T09:48:25,271 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/ns/2923e082527f492586ace479ab7f29dd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/2923e082527f492586ace479ab7f29dd 2024-11-17T09:48:25,273 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/2923e082527f492586ace479ab7f29dd, delay=931 ms 2024-11-17T09:48:25,275 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/2923e082527f492586ace479ab7f29dd, entries=2, sequenceid=15, filesize=5.0 K 2024-11-17T09:48:25,275 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/table/ba361d0eeb724cf2891dd209247ee746 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/ba361d0eeb724cf2891dd209247ee746 2024-11-17T09:48:25,277 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/ba361d0eeb724cf2891dd209247ee746, delay=950 ms 2024-11-17T09:48:25,278 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/ba361d0eeb724cf2891dd209247ee746, entries=2, sequenceid=15, filesize=5.3 K 2024-11-17T09:48:25,279 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=15, compaction requested=false 2024-11-17T09:48:25,279 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-17T09:48:25,281 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/7ec862786f1b451c9ef328796db5f0be 2024-11-17T09:48:25,283 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/2923e082527f492586ace479ab7f29dd 2024-11-17T09:48:25,284 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/ba361d0eeb724cf2891dd209247ee746 2024-11-17T09:48:25,289 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-17T09:48:25,289 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T09:48:25,290 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T09:48:25,290 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731836905171Running coprocessor pre-close hooks at 1731836905172 (+1 ms)Disabling compacts and flushes for region at 1731836905172Disabling writes for close at 1731836905172Obtaining lock to block concurrent updates at 1731836905172Preparing flush snapshotting stores in 1588230740 at 1731836905172Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731836905173 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731836905173Flushing 1588230740/info: creating writer at 1731836905173Flushing 1588230740/info: appending metadata at 1731836905202 (+29 ms)Flushing 1588230740/info: closing flushed file at 1731836905203 (+1 ms)Flushing 1588230740/ns: creating writer at 1731836905208 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731836905233 (+25 ms)Flushing 1588230740/ns: closing flushed file at 1731836905233Flushing 1588230740/table: creating writer at 1731836905240 (+7 ms)Flushing 1588230740/table: appending metadata at 1731836905262 (+22 ms)Flushing 1588230740/table: closing flushed file at 1731836905262Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6857e72f: reopening flushed file at 1731836905267 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d632402: reopening flushed file at 1731836905271 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6c0777b9: reopening flushed file at 1731836905275 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 107ms, sequenceid=15, compaction requested=false at 1731836905279 (+4 ms)Writing region close event to WAL at 1731836905284 (+5 ms)Running coprocessor post-close hooks at 1731836905289 (+5 ms)Closed at 1731836905290 (+1 ms) 2024-11-17T09:48:25,290 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T09:48:25,372 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(976): stopping server 70e37c475432,36895,1731836900525; all regions closed. 2024-11-17T09:48:25,372 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(976): stopping server 70e37c475432,37353,1731836900395; all regions closed. 2024-11-17T09:48:25,377 DEBUG [RS:1;70e37c475432:36895 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs 2024-11-17T09:48:25,377 DEBUG [RS:0;70e37c475432:37353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs 2024-11-17T09:48:25,377 INFO [RS:0;70e37c475432:37353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C37353%2C1731836900395.meta:.meta(num 1731836902373) 2024-11-17T09:48:25,377 INFO [RS:1;70e37c475432:36895 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C36895%2C1731836900525:(num 1731836902030) 2024-11-17T09:48:25,378 DEBUG [RS:1;70e37c475432:36895 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:25,378 INFO [RS:1;70e37c475432:36895 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:25,378 INFO [RS:1;70e37c475432:36895 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:48:25,378 INFO [RS:1;70e37c475432:36895 {}] hbase.ChoreService(370): Chore service for: regionserver/70e37c475432:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T09:48:25,378 INFO [RS:1;70e37c475432:36895 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T09:48:25,378 INFO [RS:1;70e37c475432:36895 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T09:48:25,378 INFO [RS:1;70e37c475432:36895 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T09:48:25,378 INFO [regionserver/70e37c475432:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:48:25,378 INFO [RS:1;70e37c475432:36895 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:48:25,379 INFO [RS:1;70e37c475432:36895 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36895 2024-11-17T09:48:25,380 DEBUG [RS:0;70e37c475432:37353 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs 2024-11-17T09:48:25,380 INFO [RS:0;70e37c475432:37353 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C37353%2C1731836900395:(num 1731836902029) 2024-11-17T09:48:25,380 DEBUG [RS:0;70e37c475432:37353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:25,380 INFO [RS:0;70e37c475432:37353 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:25,380 INFO [RS:0;70e37c475432:37353 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:48:25,381 INFO [RS:0;70e37c475432:37353 {}] hbase.ChoreService(370): Chore service for: regionserver/70e37c475432:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T09:48:25,381 INFO [RS:0;70e37c475432:37353 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:48:25,381 INFO [regionserver/70e37c475432:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:48:25,381 INFO [RS:0;70e37c475432:37353 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37353 2024-11-17T09:48:25,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/70e37c475432,36895,1731836900525 2024-11-17T09:48:25,387 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:48:25,387 INFO [RS:1;70e37c475432:36895 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:48:25,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/70e37c475432,37353,1731836900395 2024-11-17T09:48:25,388 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [70e37c475432,36895,1731836900525] 2024-11-17T09:48:25,389 INFO [RS:0;70e37c475432:37353 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:48:25,391 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/70e37c475432,36895,1731836900525 already deleted, retry=false 2024-11-17T09:48:25,392 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 70e37c475432,36895,1731836900525 expired; onlineServers=1 2024-11-17T09:48:25,392 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [70e37c475432,37353,1731836900395] 2024-11-17T09:48:25,393 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/70e37c475432,37353,1731836900395 already deleted, retry=false 2024-11-17T09:48:25,393 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 70e37c475432,37353,1731836900395 expired; onlineServers=0 2024-11-17T09:48:25,393 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '70e37c475432,32885,1731836899642' ***** 2024-11-17T09:48:25,393 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T09:48:25,393 INFO [M:0;70e37c475432:32885 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:48:25,393 INFO [M:0;70e37c475432:32885 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:48:25,394 DEBUG [M:0;70e37c475432:32885 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T09:48:25,394 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T09:48:25,394 DEBUG [M:0;70e37c475432:32885 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T09:48:25,394 DEBUG [master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836901650 {}] cleaner.HFileCleaner(306): Exit Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836901650,5,FailOnTimeoutGroup] 2024-11-17T09:48:25,394 DEBUG [master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836901645 {}] cleaner.HFileCleaner(306): Exit Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836901645,5,FailOnTimeoutGroup] 2024-11-17T09:48:25,394 INFO [M:0;70e37c475432:32885 {}] hbase.ChoreService(370): Chore service for: master/70e37c475432:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T09:48:25,394 INFO [M:0;70e37c475432:32885 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:48:25,394 DEBUG [M:0;70e37c475432:32885 {}] master.HMaster(1795): Stopping service threads 2024-11-17T09:48:25,394 INFO [M:0;70e37c475432:32885 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T09:48:25,395 INFO [M:0;70e37c475432:32885 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T09:48:25,395 INFO [M:0;70e37c475432:32885 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T09:48:25,395 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T09:48:25,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T09:48:25,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,397 DEBUG [M:0;70e37c475432:32885 {}] zookeeper.ZKUtil(347): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T09:48:25,397 WARN [M:0;70e37c475432:32885 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T09:48:25,397 INFO [M:0;70e37c475432:32885 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.lastflushedseqids 2024-11-17T09:48:25,406 INFO [M:0;70e37c475432:32885 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T09:48:25,406 INFO [M:0;70e37c475432:32885 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T09:48:25,407 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T09:48:25,407 INFO [M:0;70e37c475432:32885 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,407 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,407 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T09:48:25,407 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,407 INFO [M:0;70e37c475432:32885 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-17T09:48:25,417 DEBUG [M:0;70e37c475432:32885 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6048742965544d49a0ec2844301a92b4 2024-11-17T09:48:25,438 DEBUG [M:0;70e37c475432:32885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6048742965544d49a0ec2844301a92b4 is 82, key is hbase:meta,,1/info:regioninfo/1731836902442/Put/seqid=0 2024-11-17T09:48:25,439 INFO [M:0;70e37c475432:32885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6048742965544d49a0ec2844301a92b4 2024-11-17T09:48:25,448 DEBUG [M:0;70e37c475432:32885 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 45017a9d5a864b55b501236c2597662d 2024-11-17T09:48:25,462 DEBUG [M:0;70e37c475432:32885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45017a9d5a864b55b501236c2597662d is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731836903209/Put/seqid=0 2024-11-17T09:48:25,463 INFO [M:0;70e37c475432:32885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45017a9d5a864b55b501236c2597662d 2024-11-17T09:48:25,472 DEBUG [M:0;70e37c475432:32885 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 97d51323a6d74f12b690766b89066c0a 2024-11-17T09:48:25,484 DEBUG [M:0;70e37c475432:32885 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97d51323a6d74f12b690766b89066c0a is 69, key is 70e37c475432,36895,1731836900525/rs:state/1731836901782/Put/seqid=0 2024-11-17T09:48:25,485 INFO [M:0;70e37c475432:32885 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97d51323a6d74f12b690766b89066c0a 2024-11-17T09:48:25,487 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6048742965544d49a0ec2844301a92b4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6048742965544d49a0ec2844301a92b4 2024-11-17T09:48:25,489 INFO [M:0;70e37c475432:32885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6048742965544d49a0ec2844301a92b4, entries=8, sequenceid=84, filesize=5.5 K 2024-11-17T09:48:25,489 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/45017a9d5a864b55b501236c2597662d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45017a9d5a864b55b501236c2597662d 2024-11-17T09:48:25,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:25,490 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36895-0x1012553da900002, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:25,490 INFO [RS:1;70e37c475432:36895 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:48:25,490 INFO [RS:1;70e37c475432:36895 {}] regionserver.HRegionServer(1031): Exiting; stopping=70e37c475432,36895,1731836900525; zookeeper connection closed. 2024-11-17T09:48:25,491 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d1145c9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d1145c9 2024-11-17T09:48:25,491 INFO [M:0;70e37c475432:32885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/45017a9d5a864b55b501236c2597662d, entries=9, sequenceid=84, filesize=6.5 K 2024-11-17T09:48:25,492 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/97d51323a6d74f12b690766b89066c0a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97d51323a6d74f12b690766b89066c0a 2024-11-17T09:48:25,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:25,492 INFO [RS:0;70e37c475432:37353 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:48:25,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37353-0x1012553da900001, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:25,492 INFO [RS:0;70e37c475432:37353 {}] regionserver.HRegionServer(1031): Exiting; stopping=70e37c475432,37353,1731836900395; zookeeper connection closed. 2024-11-17T09:48:25,492 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5cbf7c40 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5cbf7c40 2024-11-17T09:48:25,493 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-17T09:48:25,493 INFO [M:0;70e37c475432:32885 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/97d51323a6d74f12b690766b89066c0a, entries=2, sequenceid=84, filesize=5.1 K 2024-11-17T09:48:25,494 INFO [M:0;70e37c475432:32885 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false 2024-11-17T09:48:25,496 INFO [M:0;70e37c475432:32885 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,497 DEBUG [M:0;70e37c475432:32885 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731836905407Disabling compacts and flushes for region at 1731836905407Disabling writes for close at 1731836905407Obtaining lock to block concurrent updates at 1731836905407Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731836905407Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731836905407Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731836905408 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731836905408Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731836905438 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731836905438Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731836905441 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731836905462 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731836905462Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731836905465 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731836905484 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731836905484Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d6c703b: reopening flushed file at 1731836905487 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d2c9de1: reopening flushed file at 1731836905489 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1dc92a2e: reopening flushed file at 1731836905491 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 87ms, sequenceid=84, compaction requested=false at 1731836905494 (+3 ms)Writing region close event to WAL at 1731836905496 (+2 ms)Closed at 1731836905496 2024-11-17T09:48:25,498 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:48:25,498 INFO [M:0;70e37c475432:32885 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T09:48:25,499 INFO [M:0;70e37c475432:32885 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32885 2024-11-17T09:48:25,499 INFO [M:0;70e37c475432:32885 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:48:25,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:25,601 INFO [M:0;70e37c475432:32885 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:48:25,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:32885-0x1012553da900000, quorum=127.0.0.1:64067, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:25,606 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T09:48:25,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-17T09:48:25,629 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,640 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=50 (was 12) Potentially hanging thread: master/70e37c475432:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/70e37c475432:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731836905093 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/70e37c475432:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/70e37c475432:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1731836905273 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1731836905269 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731836903567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1171 (was 1273), ProcessCount=11 (was 11), AvailableMemoryMB=3578 (was 3910) 2024-11-17T09:48:25,649 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=50, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=1171, ProcessCount=11, AvailableMemoryMB=3578 2024-11-17T09:48:25,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/hadoop-log-dir so I do NOT create it in target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0 2024-11-17T09:48:25,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/hadoop-tmp-dir so I do NOT create it in target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0 2024-11-17T09:48:25,650 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,653 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/cluster_b78f0e62-5554-508c-2e7b-b7742eba3210, deleteOnExit=true 2024-11-17T09:48:25,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/cluster_b78f0e62-5554-508c-2e7b-b7742eba3210/zookeeper_0, clientPort=56799, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/cluster_b78f0e62-5554-508c-2e7b-b7742eba3210/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/cluster_b78f0e62-5554-508c-2e7b-b7742eba3210/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-17T09:48:25,662 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56799 2024-11-17T09:48:25,675 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 with version=8 2024-11-17T09:48:25,675 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/hbase-staging 2024-11-17T09:48:25,678 INFO [Time-limited test {}] client.ConnectionUtils(128): master/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:48:25,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,678 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,678 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:48:25,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:48:25,679 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-17T09:48:25,679 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:48:25,679 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40981 2024-11-17T09:48:25,681 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40981 connecting to ZooKeeper ensemble=127.0.0.1:56799 2024-11-17T09:48:25,687 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:409810x0, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:48:25,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40981-0x1012553f3470000 connected 2024-11-17T09:48:25,703 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,703 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,704 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:25,705 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8, hbase.cluster.distributed=false 2024-11-17T09:48:25,707 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:48:25,708 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40981 2024-11-17T09:48:25,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40981 2024-11-17T09:48:25,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40981 2024-11-17T09:48:25,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40981 2024-11-17T09:48:25,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40981 2024-11-17T09:48:25,728 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:48:25,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,728 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:48:25,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,728 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:48:25,728 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T09:48:25,728 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:48:25,732 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39601 2024-11-17T09:48:25,734 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39601 connecting to ZooKeeper ensemble=127.0.0.1:56799 2024-11-17T09:48:25,734 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,734 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:396010x0, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:48:25,741 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:396010x0, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:25,742 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39601-0x1012553f3470001 connected 2024-11-17T09:48:25,742 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T09:48:25,745 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.cache 2024-11-17T09:48:25,745 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T09:48:25,745 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T09:48:25,745 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T09:48:25,747 INFO [Thread-118 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T09:48:25,747 WARN [Thread-118 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T09:48:25,748 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T09:48:25,748 INFO [Thread-118 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence is complete. 2024-11-17T09:48:25,749 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T09:48:25,751 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T09:48:25,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T09:48:25,753 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:48:25,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39601 2024-11-17T09:48:25,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39601 2024-11-17T09:48:25,757 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39601 2024-11-17T09:48:25,760 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39601 2024-11-17T09:48:25,761 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39601 2024-11-17T09:48:25,778 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:48:25,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,779 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:48:25,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:25,779 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:48:25,779 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T09:48:25,779 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:48:25,780 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43629 2024-11-17T09:48:25,781 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43629 connecting to ZooKeeper ensemble=127.0.0.1:56799 2024-11-17T09:48:25,781 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,782 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,784 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:436290x0, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:48:25,785 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43629-0x1012553f3470002 connected 2024-11-17T09:48:25,785 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:25,785 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T09:48:25,788 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.cache 2024-11-17T09:48:25,789 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T09:48:25,789 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T09:48:25,789 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T09:48:25,789 INFO [Thread-119 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T09:48:25,789 WARN [Thread-119 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-17T09:48:25,789 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T09:48:25,790 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T09:48:25,790 INFO [Thread-119 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence is complete. 2024-11-17T09:48:25,790 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T09:48:25,791 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-17T09:48:25,792 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:48:25,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43629 2024-11-17T09:48:25,796 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43629 2024-11-17T09:48:25,801 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43629 2024-11-17T09:48:25,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43629 2024-11-17T09:48:25,802 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43629 2024-11-17T09:48:25,814 DEBUG [M:0;70e37c475432:40981 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;70e37c475432:40981 2024-11-17T09:48:25,815 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/70e37c475432,40981,1731836905677 2024-11-17T09:48:25,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:25,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:25,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:25,819 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/70e37c475432,40981,1731836905677 2024-11-17T09:48:25,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T09:48:25,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-17T09:48:25,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,821 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,822 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,823 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T09:48:25,823 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/70e37c475432,40981,1731836905677 from backup master directory 2024-11-17T09:48:25,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:25,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/70e37c475432,40981,1731836905677 2024-11-17T09:48:25,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:25,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-17T09:48:25,825 WARN [master/70e37c475432:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:48:25,825 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=70e37c475432,40981,1731836905677 2024-11-17T09:48:25,827 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/hbase.id] with ID: e242b801-37af-4aed-a76c-156695abeb2f 2024-11-17T09:48:25,827 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.tmp/hbase.id 2024-11-17T09:48:25,836 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/hbase.id] 2024-11-17T09:48:25,876 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:25,876 INFO [master/70e37c475432:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-17T09:48:25,878 INFO [master/70e37c475432:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-17T09:48:25,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,881 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:25,886 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:48:25,888 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-17T09:48:25,888 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:25,909 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store 2024-11-17T09:48:25,915 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:25,915 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T09:48:25,915 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,916 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,916 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T09:48:25,916 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,916 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:25,916 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731836905915Disabling compacts and flushes for region at 1731836905915Disabling writes for close at 1731836905916 (+1 ms)Writing region close event to WAL at 1731836905916Closed at 1731836905916 2024-11-17T09:48:25,916 WARN [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/.initializing 2024-11-17T09:48:25,916 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/WALs/70e37c475432,40981,1731836905677 2024-11-17T09:48:25,922 INFO [master/70e37c475432:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:48:25,926 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=70e37c475432%2C40981%2C1731836905677, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/WALs/70e37c475432,40981,1731836905677, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/oldWALs, maxLogs=10 2024-11-17T09:48:25,950 INFO [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/WALs/70e37c475432,40981,1731836905677/70e37c475432%2C40981%2C1731836905677.1731836905926 2024-11-17T09:48:25,950 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:25,950 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:25,951 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:25,951 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,951 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,952 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-17T09:48:25,957 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:25,957 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:25,958 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-17T09:48:25,961 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:25,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:25,961 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-17T09:48:25,965 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:25,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:25,965 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-17T09:48:25,969 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:25,969 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:25,969 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,970 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,970 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,970 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,970 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,970 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T09:48:25,971 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-17T09:48:25,977 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:48:25,977 INFO [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61264361, jitterRate=-0.08708988130092621}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T09:48:25,978 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731836905951Initializing all the Stores at 1731836905951Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836905951Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836905952 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836905952Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836905952Cleaning up temporary data from old regions at 1731836905970 (+18 ms)Region opened successfully at 1731836905977 (+7 ms) 2024-11-17T09:48:25,978 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-17T09:48:25,982 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c25b7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:48:25,983 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-17T09:48:25,984 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-17T09:48:25,984 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-17T09:48:25,984 INFO [master/70e37c475432:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-17T09:48:25,984 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-17T09:48:25,985 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-17T09:48:25,985 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-17T09:48:25,986 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-17T09:48:25,987 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-17T09:48:25,989 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-17T09:48:25,990 INFO [master/70e37c475432:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-17T09:48:25,990 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-17T09:48:25,991 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-17T09:48:25,992 INFO [master/70e37c475432:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-17T09:48:25,993 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-17T09:48:25,994 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-17T09:48:25,995 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-17T09:48:25,996 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-17T09:48:25,998 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-17T09:48:26,000 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-17T09:48:26,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:26,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:26,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:26,002 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,003 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=70e37c475432,40981,1731836905677, sessionid=0x1012553f3470000, setting cluster-up flag (Was=false) 2024-11-17T09:48:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,011 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-17T09:48:26,012 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=70e37c475432,40981,1731836905677 2024-11-17T09:48:26,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,020 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-17T09:48:26,022 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=70e37c475432,40981,1731836905677 2024-11-17T09:48:26,022 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-17T09:48:26,024 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:26,024 INFO [master/70e37c475432:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-17T09:48:26,024 INFO [master/70e37c475432:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-17T09:48:26,025 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 70e37c475432,40981,1731836905677 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-17T09:48:26,026 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:26,026 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:26,026 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:26,026 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/70e37c475432:0, corePoolSize=5, maxPoolSize=5 2024-11-17T09:48:26,027 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/70e37c475432:0, corePoolSize=10, maxPoolSize=10 2024-11-17T09:48:26,027 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,027 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:48:26,027 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,028 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:26,028 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-17T09:48:26,029 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,029 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-17T09:48:26,030 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731836936030 2024-11-17T09:48:26,030 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-17T09:48:26,030 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-17T09:48:26,030 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-17T09:48:26,030 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-17T09:48:26,030 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-17T09:48:26,030 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-17T09:48:26,031 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,031 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-17T09:48:26,031 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-17T09:48:26,031 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-17T09:48:26,031 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-17T09:48:26,032 INFO [master/70e37c475432:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-17T09:48:26,032 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836906032,5,FailOnTimeoutGroup] 2024-11-17T09:48:26,032 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836906032,5,FailOnTimeoutGroup] 2024-11-17T09:48:26,032 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,033 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-17T09:48:26,033 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,033 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,042 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-17T09:48:26,042 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:26,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:26,049 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T09:48:26,052 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T09:48:26,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,052 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T09:48:26,056 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T09:48:26,056 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T09:48:26,059 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T09:48:26,059 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,059 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T09:48:26,063 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T09:48:26,063 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T09:48:26,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:26,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:26,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T09:48:26,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T09:48:26,064 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T09:48:26,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T09:48:26,070 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:48:26,071 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68159792, jitterRate=0.01566004753112793}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T09:48:26,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731836906048Initializing all the Stores at 1731836906048Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836906049 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836906049Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836906049Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836906049Cleaning up temporary data from old regions at 1731836906064 (+15 ms)Region opened successfully at 1731836906071 (+7 ms) 2024-11-17T09:48:26,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T09:48:26,071 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T09:48:26,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T09:48:26,071 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T09:48:26,072 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T09:48:26,072 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T09:48:26,072 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731836906071Disabling compacts and flushes for region at 1731836906071Disabling writes for close at 1731836906071Writing region close event to WAL at 1731836906072 (+1 ms)Closed at 1731836906072 2024-11-17T09:48:26,073 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:26,073 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-17T09:48:26,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-17T09:48:26,074 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T09:48:26,075 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-17T09:48:26,104 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(746): ClusterId : e242b801-37af-4aed-a76c-156695abeb2f 2024-11-17T09:48:26,105 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(746): ClusterId : e242b801-37af-4aed-a76c-156695abeb2f 2024-11-17T09:48:26,105 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T09:48:26,105 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T09:48:26,108 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T09:48:26,108 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T09:48:26,108 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T09:48:26,108 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T09:48:26,110 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T09:48:26,110 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T09:48:26,110 DEBUG [RS:0;70e37c475432:39601 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4538eb92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:48:26,111 DEBUG [RS:1;70e37c475432:43629 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@592c5be4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:48:26,123 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;70e37c475432:43629 2024-11-17T09:48:26,123 INFO [RS:1;70e37c475432:43629 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T09:48:26,123 INFO [RS:1;70e37c475432:43629 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T09:48:26,123 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T09:48:26,123 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;70e37c475432:39601 2024-11-17T09:48:26,123 INFO [RS:0;70e37c475432:39601 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T09:48:26,123 INFO [RS:0;70e37c475432:39601 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T09:48:26,123 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T09:48:26,124 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,40981,1731836905677 with port=43629, startcode=1731836905778 2024-11-17T09:48:26,124 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,40981,1731836905677 with port=39601, startcode=1731836905727 2024-11-17T09:48:26,124 DEBUG [RS:0;70e37c475432:39601 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T09:48:26,124 DEBUG [RS:1;70e37c475432:43629 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T09:48:26,126 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55909, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T09:48:26,126 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37125, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T09:48:26,127 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40981 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 70e37c475432,43629,1731836905778 2024-11-17T09:48:26,127 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40981 {}] master.ServerManager(517): Registering regionserver=70e37c475432,43629,1731836905778 2024-11-17T09:48:26,129 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40981 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 70e37c475432,39601,1731836905727 2024-11-17T09:48:26,129 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40981 {}] master.ServerManager(517): Registering regionserver=70e37c475432,39601,1731836905727 2024-11-17T09:48:26,129 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:26,129 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T09:48:26,129 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T09:48:26,130 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:26,130 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T09:48:26,130 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T09:48:26,131 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:48:26,138 DEBUG [RS:1;70e37c475432:43629 {}] zookeeper.ZKUtil(111): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/70e37c475432,43629,1731836905778 2024-11-17T09:48:26,139 WARN [RS:1;70e37c475432:43629 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:48:26,139 INFO [RS:1;70e37c475432:43629 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:26,139 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [70e37c475432,43629,1731836905778] 2024-11-17T09:48:26,139 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [70e37c475432,39601,1731836905727] 2024-11-17T09:48:26,139 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,43629,1731836905778 2024-11-17T09:48:26,139 DEBUG [RS:0;70e37c475432:39601 {}] zookeeper.ZKUtil(111): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/70e37c475432,39601,1731836905727 2024-11-17T09:48:26,139 WARN [RS:0;70e37c475432:39601 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:48:26,139 INFO [RS:0;70e37c475432:39601 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:26,139 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,39601,1731836905727 2024-11-17T09:48:26,144 INFO [RS:0;70e37c475432:39601 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T09:48:26,144 INFO [RS:1;70e37c475432:43629 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T09:48:26,146 INFO [RS:0;70e37c475432:39601 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T09:48:26,147 INFO [RS:0;70e37c475432:39601 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T09:48:26,147 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,147 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T09:48:26,148 INFO [RS:0;70e37c475432:39601 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T09:48:26,148 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,148 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,149 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,149 INFO [RS:1;70e37c475432:43629 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T09:48:26,149 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,149 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,149 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,149 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:26,149 DEBUG [RS:0;70e37c475432:39601 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:26,152 INFO [RS:1;70e37c475432:43629 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T09:48:26,152 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,153 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,153 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,153 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T09:48:26,153 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,153 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,153 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,153 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,39601,1731836905727-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:48:26,153 INFO [RS:1;70e37c475432:43629 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T09:48:26,153 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,154 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,155 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,155 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:26,155 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:26,155 DEBUG [RS:1;70e37c475432:43629 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:26,156 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,156 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,156 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,156 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,156 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,156 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,43629,1731836905778-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:48:26,172 INFO [RS:1;70e37c475432:43629 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T09:48:26,172 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,43629,1731836905778-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,172 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,172 INFO [RS:1;70e37c475432:43629 {}] regionserver.Replication(171): 70e37c475432,43629,1731836905778 started 2024-11-17T09:48:26,175 INFO [RS:0;70e37c475432:39601 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T09:48:26,175 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,39601,1731836905727-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,176 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,176 INFO [RS:0;70e37c475432:39601 {}] regionserver.Replication(171): 70e37c475432,39601,1731836905727 started 2024-11-17T09:48:26,187 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,187 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1482): Serving as 70e37c475432,43629,1731836905778, RpcServer on 70e37c475432/172.17.0.2:43629, sessionid=0x1012553f3470002 2024-11-17T09:48:26,187 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T09:48:26,187 DEBUG [RS:1;70e37c475432:43629 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 70e37c475432,43629,1731836905778 2024-11-17T09:48:26,187 DEBUG [RS:1;70e37c475432:43629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,43629,1731836905778' 2024-11-17T09:48:26,187 DEBUG [RS:1;70e37c475432:43629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T09:48:26,188 DEBUG [RS:1;70e37c475432:43629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T09:48:26,188 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T09:48:26,188 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T09:48:26,188 DEBUG [RS:1;70e37c475432:43629 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 70e37c475432,43629,1731836905778 2024-11-17T09:48:26,189 DEBUG [RS:1;70e37c475432:43629 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,43629,1731836905778' 2024-11-17T09:48:26,189 DEBUG [RS:1;70e37c475432:43629 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T09:48:26,189 DEBUG [RS:1;70e37c475432:43629 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T09:48:26,190 DEBUG [RS:1;70e37c475432:43629 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T09:48:26,190 INFO [RS:1;70e37c475432:43629 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T09:48:26,190 INFO [RS:1;70e37c475432:43629 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T09:48:26,191 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,191 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1482): Serving as 70e37c475432,39601,1731836905727, RpcServer on 70e37c475432/172.17.0.2:39601, sessionid=0x1012553f3470001 2024-11-17T09:48:26,192 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T09:48:26,192 DEBUG [RS:0;70e37c475432:39601 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 70e37c475432,39601,1731836905727 2024-11-17T09:48:26,192 DEBUG [RS:0;70e37c475432:39601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,39601,1731836905727' 2024-11-17T09:48:26,192 DEBUG [RS:0;70e37c475432:39601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T09:48:26,192 DEBUG [RS:0;70e37c475432:39601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T09:48:26,193 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T09:48:26,193 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T09:48:26,193 DEBUG [RS:0;70e37c475432:39601 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 70e37c475432,39601,1731836905727 2024-11-17T09:48:26,193 DEBUG [RS:0;70e37c475432:39601 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,39601,1731836905727' 2024-11-17T09:48:26,193 DEBUG [RS:0;70e37c475432:39601 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T09:48:26,193 DEBUG [RS:0;70e37c475432:39601 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T09:48:26,194 DEBUG [RS:0;70e37c475432:39601 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T09:48:26,194 INFO [RS:0;70e37c475432:39601 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T09:48:26,194 INFO [RS:0;70e37c475432:39601 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T09:48:26,225 WARN [70e37c475432:40981 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-17T09:48:26,291 INFO [RS:1;70e37c475432:43629 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:48:26,292 INFO [RS:1;70e37c475432:43629 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=70e37c475432%2C43629%2C1731836905778, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,43629,1731836905778, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs, maxLogs=55 2024-11-17T09:48:26,294 INFO [RS:0;70e37c475432:39601 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:48:26,295 INFO [RS:0;70e37c475432:39601 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=70e37c475432%2C39601%2C1731836905727, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,39601,1731836905727, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs, maxLogs=55 2024-11-17T09:48:26,312 INFO [RS:1;70e37c475432:43629 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,43629,1731836905778/70e37c475432%2C43629%2C1731836905778.1731836906294 2024-11-17T09:48:26,312 DEBUG [RS:1;70e37c475432:43629 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:26,313 INFO [RS:0;70e37c475432:39601 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,39601,1731836905727/70e37c475432%2C39601%2C1731836905727.1731836906296 2024-11-17T09:48:26,313 DEBUG [RS:0;70e37c475432:39601 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:26,475 DEBUG [70e37c475432:40981 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-17T09:48:26,476 DEBUG [70e37c475432:40981 {}] balancer.BalancerClusterState(204): Hosts are {70e37c475432=0} racks are {/default-rack=0} 2024-11-17T09:48:26,477 DEBUG [70e37c475432:40981 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T09:48:26,477 DEBUG [70e37c475432:40981 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T09:48:26,477 DEBUG [70e37c475432:40981 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T09:48:26,477 DEBUG [70e37c475432:40981 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T09:48:26,477 INFO [70e37c475432:40981 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T09:48:26,477 INFO [70e37c475432:40981 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T09:48:26,477 DEBUG [70e37c475432:40981 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T09:48:26,477 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=70e37c475432,39601,1731836905727 2024-11-17T09:48:26,478 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 70e37c475432,39601,1731836905727, state=OPENING 2024-11-17T09:48:26,480 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-17T09:48:26,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,481 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:26,482 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-17T09:48:26,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:26,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:26,482 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=70e37c475432,39601,1731836905727}] 2024-11-17T09:48:26,482 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:26,635 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T09:48:26,637 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46295, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T09:48:26,639 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-17T09:48:26,639 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:26,639 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-17T09:48:26,640 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=70e37c475432%2C39601%2C1731836905727.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,39601,1731836905727, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs, maxLogs=55 2024-11-17T09:48:26,659 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,39601,1731836905727/70e37c475432%2C39601%2C1731836905727.meta.1731836906641.meta 2024-11-17T09:48:26,660 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:26,660 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:26,660 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-17T09:48:26,660 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-17T09:48:26,661 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-17T09:48:26,661 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-17T09:48:26,661 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:26,661 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-17T09:48:26,661 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-17T09:48:26,662 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-17T09:48:26,662 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-17T09:48:26,662 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,663 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-17T09:48:26,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-17T09:48:26,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-17T09:48:26,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-17T09:48:26,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-17T09:48:26,665 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-17T09:48:26,665 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,666 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-17T09:48:26,666 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-17T09:48:26,666 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:26,667 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740 2024-11-17T09:48:26,667 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-17T09:48:26,667 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-17T09:48:26,667 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-17T09:48:26,668 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-17T09:48:26,669 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60577281, jitterRate=-0.09732817113399506}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-17T09:48:26,669 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-17T09:48:26,669 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731836906661Writing region info on filesystem at 1731836906661Initializing all the Stores at 1731836906661Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836906661Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836906662 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836906662Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731836906662Cleaning up temporary data from old regions at 1731836906667 (+5 ms)Running coprocessor post-open hooks at 1731836906669 (+2 ms)Region opened successfully at 1731836906669 2024-11-17T09:48:26,670 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731836906635 2024-11-17T09:48:26,673 DEBUG [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-17T09:48:26,673 INFO [RS_OPEN_META-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-17T09:48:26,673 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,39601,1731836905727 2024-11-17T09:48:26,674 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 70e37c475432,39601,1731836905727, state=OPEN 2024-11-17T09:48:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:48:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:48:26,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-17T09:48:26,677 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=70e37c475432,39601,1731836905727 2024-11-17T09:48:26,677 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:26,677 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:26,677 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-17T09:48:26,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-17T09:48:26,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=70e37c475432,39601,1731836905727 in 195 msec 2024-11-17T09:48:26,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-17T09:48:26,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 606 msec 2024-11-17T09:48:26,682 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-17T09:48:26,682 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-17T09:48:26,685 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:48:26,685 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,39601,1731836905727, seqNum=-1] 2024-11-17T09:48:26,685 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:48:26,687 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59499, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:48:26,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 668 msec 2024-11-17T09:48:26,692 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731836906692, completionTime=-1 2024-11-17T09:48:26,692 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-17T09:48:26,692 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-17T09:48:26,694 INFO [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-17T09:48:26,694 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731836966694 2024-11-17T09:48:26,694 INFO [master/70e37c475432:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731837026694 2024-11-17T09:48:26,694 INFO [master/70e37c475432:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-17T09:48:26,695 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40981,1731836905677-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,695 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40981,1731836905677-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,695 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40981,1731836905677-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,695 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-70e37c475432:40981, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,695 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,695 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,697 DEBUG [master/70e37c475432:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-17T09:48:26,699 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.874sec 2024-11-17T09:48:26,700 INFO [master/70e37c475432:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-17T09:48:26,700 INFO [master/70e37c475432:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-17T09:48:26,700 INFO [master/70e37c475432:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-17T09:48:26,700 INFO [master/70e37c475432:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-17T09:48:26,700 INFO [master/70e37c475432:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-17T09:48:26,700 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40981,1731836905677-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:48:26,700 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40981,1731836905677-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-17T09:48:26,701 DEBUG [master/70e37c475432:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-17T09:48:26,701 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-17T09:48:26,701 INFO [master/70e37c475432:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40981,1731836905677-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:26,705 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:48:26,705 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 70e37c475432,40981,-1 for getting cluster id 2024-11-17T09:48:26,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-17T09:48:26,707 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e242b801-37af-4aed-a76c-156695abeb2f' 2024-11-17T09:48:26,707 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-17T09:48:26,707 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e242b801-37af-4aed-a76c-156695abeb2f" 2024-11-17T09:48:26,708 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ed63546, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:48:26,708 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [70e37c475432,40981,-1] 2024-11-17T09:48:26,708 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-17T09:48:26,708 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:26,710 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-17T09:48:26,711 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-17T09:48:26,711 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-17T09:48:26,713 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=70e37c475432,39601,1731836905727, seqNum=-1] 2024-11-17T09:48:26,713 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:48:26,715 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:48:26,717 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=70e37c475432,40981,1731836905677 2024-11-17T09:48:26,717 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-17T09:48:26,718 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 70e37c475432,40981,1731836905677 2024-11-17T09:48:26,719 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c57bde7 2024-11-17T09:48:26,719 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-17T09:48:26,720 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-17T09:48:26,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731836906717', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-17T09:48:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 2024-11-17T09:48:26,723 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-17T09:48:26,724 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:26,724 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731836906717" procId is: 4 2024-11-17T09:48:26,725 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-17T09:48:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:26,736 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 21e9bd53ef6be3f74e171f1e8af18ff4, NAME => 'testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731836906717', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:26,742 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:26,742 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(1722): Closing 21e9bd53ef6be3f74e171f1e8af18ff4, disabling compactions & flushes 2024-11-17T09:48:26,742 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:26,742 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:26,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. after waiting 0 ms 2024-11-17T09:48:26,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:26,743 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:26,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731836906717-pool-0 {}] regionserver.HRegion(1676): Region close journal for 21e9bd53ef6be3f74e171f1e8af18ff4: Waiting for close lock at 1731836906742Disabling compacts and flushes for region at 1731836906742Disabling writes for close at 1731836906743 (+1 ms)Writing region close event to WAL at 1731836906743Closed at 1731836906743 2024-11-17T09:48:26,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 execute state=CREATE_TABLE_ADD_TO_META 2024-11-17T09:48:26,744 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731836906744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731836906744"}]},"ts":"1731836906744"} 2024-11-17T09:48:26,747 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-17T09:48:26,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-17T09:48:26,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731836906717","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836906748"}]},"ts":"1731836906748"} 2024-11-17T09:48:26,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731836906717, state=ENABLING in hbase:meta 2024-11-17T09:48:26,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {70e37c475432=0} racks are {/default-rack=0} 2024-11-17T09:48:26,752 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-17T09:48:26,752 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-17T09:48:26,752 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-17T09:48:26,752 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-17T09:48:26,752 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-17T09:48:26,752 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-17T09:48:26,752 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-17T09:48:26,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN}] 2024-11-17T09:48:26,754 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN 2024-11-17T09:48:26,754 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN; state=OFFLINE, location=70e37c475432,43629,1731836905778; forceNewPlan=false, retain=false 2024-11-17T09:48:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:26,905 INFO [70e37c475432:40981 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-17T09:48:26,906 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21e9bd53ef6be3f74e171f1e8af18ff4, regionState=OPENING, regionLocation=70e37c475432,43629,1731836905778 2024-11-17T09:48:26,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN because future has completed 2024-11-17T09:48:26,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e9bd53ef6be3f74e171f1e8af18ff4, server=70e37c475432,43629,1731836905778}] 2024-11-17T09:48:27,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:27,062 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-17T09:48:27,064 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50437, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-17T09:48:27,067 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:27,067 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 21e9bd53ef6be3f74e171f1e8af18ff4, NAME => 'testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:27,068 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731836906717 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,068 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:27,068 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,068 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,069 INFO [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,073 INFO [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21e9bd53ef6be3f74e171f1e8af18ff4 columnFamilyName family 2024-11-17T09:48:27,073 DEBUG [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:27,073 INFO [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] regionserver.HStore(327): Store=21e9bd53ef6be3f74e171f1e8af18ff4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:27,074 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,074 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,074 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,075 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,075 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,076 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,082 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-17T09:48:27,082 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 21e9bd53ef6be3f74e171f1e8af18ff4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68839056, jitterRate=0.025781869888305664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T09:48:27,083 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:27,083 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 21e9bd53ef6be3f74e171f1e8af18ff4: Running coprocessor pre-open hook at 1731836907068Writing region info on filesystem at 1731836907068Initializing all the Stores at 1731836907068Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836907068Cleaning up temporary data from old regions at 1731836907075 (+7 ms)Running coprocessor post-open hooks at 1731836907083 (+8 ms)Region opened successfully at 1731836907083 2024-11-17T09:48:27,084 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4., pid=6, masterSystemTime=1731836907062 2024-11-17T09:48:27,087 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:27,087 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:27,088 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=21e9bd53ef6be3f74e171f1e8af18ff4, regionState=OPEN, openSeqNum=2, regionLocation=70e37c475432,43629,1731836905778 2024-11-17T09:48:27,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e9bd53ef6be3f74e171f1e8af18ff4, server=70e37c475432,43629,1731836905778 because future has completed 2024-11-17T09:48:27,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-17T09:48:27,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 21e9bd53ef6be3f74e171f1e8af18ff4, server=70e37c475432,43629,1731836905778 in 183 msec 2024-11-17T09:48:27,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-17T09:48:27,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN in 342 msec 2024-11-17T09:48:27,096 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-17T09:48:27,097 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731836906717","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731836907096"}]},"ts":"1731836907096"} 2024-11-17T09:48:27,099 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731836906717, state=ENABLED in hbase:meta 2024-11-17T09:48:27,099 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 execute state=CREATE_TABLE_POST_OPERATION 2024-11-17T09:48:27,101 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731836906717 in 379 msec 2024-11-17T09:48:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40981 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-17T09:48:27,357 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731836906717 get assigned. Timeout = 60000ms 2024-11-17T09:48:27,357 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731836906717 completed 2024-11-17T09:48:27,357 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T09:48:27,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731836906717 assigned to meta. Checking AM states. 2024-11-17T09:48:27,361 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-17T09:48:27,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731836906717 assigned. 2024-11-17T09:48:27,364 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731836906717', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4., hostname=70e37c475432,43629,1731836905778, seqNum=2] 2024-11-17T09:48:27,365 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-17T09:48:27,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-17T09:48:27,371 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 21e9bd53ef6be3f74e171f1e8af18ff4 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-17T09:48:27,381 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ee52269334d549a1a1d9febddb7ff78c 2024-11-17T09:48:27,394 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/.tmp/family/ee52269334d549a1a1d9febddb7ff78c is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-17T09:48:27,395 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/.tmp/family/ee52269334d549a1a1d9febddb7ff78c 2024-11-17T09:48:27,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/.tmp/family/ee52269334d549a1a1d9febddb7ff78c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c 2024-11-17T09:48:27,399 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c, delay=911 ms 2024-11-17T09:48:27,400 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c, entries=2, sequenceid=6, filesize=4.9 K 2024-11-17T09:48:27,401 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 21e9bd53ef6be3f74e171f1e8af18ff4 in 30ms, sequenceid=6, compaction requested=false 2024-11-17T09:48:27,401 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731836906717' 2024-11-17T09:48:27,402 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 21e9bd53ef6be3f74e171f1e8af18ff4: 2024-11-17T09:48:27,912 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-17T09:48:28,311 DEBUG [hfile-prefetch-1731836905093 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ee52269334d549a1a1d9febddb7ff78c_0. 2024-11-17T09:48:28,312 DEBUG [hfile-prefetch-1731836905093 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key ee52269334d549a1a1d9febddb7ff78c_117. 2024-11-17T09:48:28,312 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c, with total blocks 2, and data blocks 1 2024-11-17T09:48:28,312 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-17T09:48:28,312 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-17T09:48:28,312 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c. Will sleep 100ms and try the verification again. 2024-11-17T09:48:28,348 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-17T09:48:28,348 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-17T09:48:28,349 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:28,349 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T09:48:28,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T09:48:28,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731836908348 2024-11-17T09:48:28,352 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-17T09:48:28,402 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;70e37c475432:43629,5,FailOnTimeoutGroup] 2024-11-17T09:48:28,402 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '70e37c475432,43629,1731836905778' ***** 2024-11-17T09:48:28,402 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-17T09:48:28,403 INFO [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T09:48:28,403 INFO [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence 2024-11-17T09:48:28,403 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T09:48:28,403 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T09:48:28,403 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T09:48:28,403 INFO [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T09:48:28,403 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T09:48:28,412 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c, with total blocks 2, and data blocks 1 2024-11-17T09:48:28,412 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-17T09:48:28,412 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2293): found block for file ee52269334d549a1a1d9febddb7ff78c in the backing map. Acquiring read lock for offset 0 2024-11-17T09:48:28,412 DEBUG [hfile-prefetch-1731836905093 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c has now been fully cached. 2024-11-17T09:48:28,413 DEBUG [hfile-prefetch-1731836905093 {}] hfile.PrefetchExecutor(121): Prefetch completed for ee52269334d549a1a1d9febddb7ff78c 2024-11-17T09:48:28,413 DEBUG [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1487): Thread RS:1;70e37c475432:43629 started persisting bucket cache to file 2024-11-17T09:48:28,413 DEBUG [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:28,413 DEBUG [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-17T09:48:28,421 DEBUG [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-17T09:48:28,421 DEBUG [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731836908413 2024-11-17T09:48:28,421 DEBUG [RS:1;70e37c475432:43629 {}] bucket.BucketCache(1509): Thread RS:1;70e37c475432:43629 finished persisting bucket cache to file, renaming 2024-11-17T09:48:28,421 INFO [RS:1;70e37c475432:43629 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T09:48:28,421 INFO [RS:1;70e37c475432:43629 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T09:48:28,421 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T09:48:28,422 INFO [RS:1;70e37c475432:43629 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T09:48:28,422 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(3091): Received CLOSE for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,422 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(959): stopping server 70e37c475432,43629,1731836905778 2024-11-17T09:48:28,422 INFO [RS:1;70e37c475432:43629 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:48:28,422 INFO [RS:1;70e37c475432:43629 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;70e37c475432:43629. 2024-11-17T09:48:28,422 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T09:48:28,422 DEBUG [RS:1;70e37c475432:43629 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:28,422 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 21e9bd53ef6be3f74e171f1e8af18ff4, disabling compactions & flushes 2024-11-17T09:48:28,422 DEBUG [RS:1;70e37c475432:43629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:28,422 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,422 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,422 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. after waiting 0 ms 2024-11-17T09:48:28,422 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-17T09:48:28,422 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,422 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1325): Online Regions={21e9bd53ef6be3f74e171f1e8af18ff4=testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.} 2024-11-17T09:48:28,422 DEBUG [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1351): Waiting on 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,427 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-17T09:48:28,428 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,428 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 21e9bd53ef6be3f74e171f1e8af18ff4: Waiting for close lock at 1731836908422Running coprocessor pre-close hooks at 1731836908422Disabling compacts and flushes for region at 1731836908422Disabling writes for close at 1731836908422Writing region close event to WAL at 1731836908423 (+1 ms)Running coprocessor post-close hooks at 1731836908427 (+4 ms)Closed at 1731836908427 2024-11-17T09:48:28,428 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,458 INFO [regionserver/70e37c475432:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:28,623 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(976): stopping server 70e37c475432,43629,1731836905778; all regions closed. 2024-11-17T09:48:28,625 DEBUG [RS:1;70e37c475432:43629 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C43629%2C1731836905778:(num 1731836906294) 2024-11-17T09:48:28,625 DEBUG [RS:1;70e37c475432:43629 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] hbase.ChoreService(370): Chore service for: regionserver/70e37c475432:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:48:28,625 INFO [regionserver/70e37c475432:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:48:28,625 INFO [RS:1;70e37c475432:43629 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43629 2024-11-17T09:48:28,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/70e37c475432,43629,1731836905778 2024-11-17T09:48:28,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:48:28,629 INFO [RS:1;70e37c475432:43629 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:48:28,632 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [70e37c475432,43629,1731836905778] 2024-11-17T09:48:28,634 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/70e37c475432,43629,1731836905778 already deleted, retry=false 2024-11-17T09:48:28,634 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 70e37c475432,43629,1731836905778 on 70e37c475432,40981,1731836905677 2024-11-17T09:48:28,638 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 70e37c475432,43629,1731836905778, splitWal=true, meta=false 2024-11-17T09:48:28,641 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 70e37c475432,43629,1731836905778 (carryingMeta=false) 70e37c475432,43629,1731836905778/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2c8de290[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-17T09:48:28,641 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 70e37c475432,43629,1731836905778, splitWal=true, meta=false 2024-11-17T09:48:28,643 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 70e37c475432,43629,1731836905778 had 1 regions 2024-11-17T09:48:28,644 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 70e37c475432,43629,1731836905778, splitWal=true, meta=false, isMeta: false 2024-11-17T09:48:28,644 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,43629,1731836905778-splitting 2024-11-17T09:48:28,645 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,43629,1731836905778-splitting dir is empty, no logs to split. 2024-11-17T09:48:28,645 INFO [PEWorker-2 {}] master.SplitWALManager(105): 70e37c475432,43629,1731836905778 WAL count=0, meta=false 2024-11-17T09:48:28,647 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,43629,1731836905778-splitting dir is empty, no logs to split. 2024-11-17T09:48:28,647 INFO [PEWorker-2 {}] master.SplitWALManager(105): 70e37c475432,43629,1731836905778 WAL count=0, meta=false 2024-11-17T09:48:28,647 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 70e37c475432,43629,1731836905778 WAL splitting is done? wals=0, meta=false 2024-11-17T09:48:28,648 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 70e37c475432,43629,1731836905778 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,43629,1731836905778-splitting does not exist 2024-11-17T09:48:28,650 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN}] 2024-11-17T09:48:28,651 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN 2024-11-17T09:48:28,651 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-17T09:48:28,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:28,732 INFO [RS:1;70e37c475432:43629 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:48:28,733 INFO [RS:1;70e37c475432:43629 {}] regionserver.HRegionServer(1031): Exiting; stopping=70e37c475432,43629,1731836905778; zookeeper connection closed. 2024-11-17T09:48:28,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43629-0x1012553f3470002, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:28,733 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5487e362 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5487e362 2024-11-17T09:48:28,802 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=21e9bd53ef6be3f74e171f1e8af18ff4, regionState=OPENING, regionLocation=70e37c475432,39601,1731836905727 2024-11-17T09:48:28,805 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN because future has completed 2024-11-17T09:48:28,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e9bd53ef6be3f74e171f1e8af18ff4, server=70e37c475432,39601,1731836905727}] 2024-11-17T09:48:28,918 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/70e37c475432:0 server-side Connection retries=45 2024-11-17T09:48:28,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:28,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:28,919 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-17T09:48:28,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-17T09:48:28,919 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-17T09:48:28,919 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-17T09:48:28,919 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-17T09:48:28,920 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40129 2024-11-17T09:48:28,921 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40129 connecting to ZooKeeper ensemble=127.0.0.1:56799 2024-11-17T09:48:28,921 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:28,921 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:28,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:401290x0, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-17T09:48:28,924 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40129-0x1012553f3470003 connected 2024-11-17T09:48:28,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-17T09:48:28,925 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-17T09:48:28,925 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-17T09:48:28,926 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.cache 2024-11-17T09:48:28,926 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-17T09:48:28,926 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-17T09:48:28,926 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-17T09:48:28,926 INFO [Thread-199 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-17T09:48:28,926 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-17T09:48:28,927 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-17T09:48:28,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-17T09:48:28,927 INFO [Thread-199 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-17T09:48:28,929 INFO [Thread-199 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-17T09:48:28,929 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-17T09:48:28,931 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40129 2024-11-17T09:48:28,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40129 2024-11-17T09:48:28,932 DEBUG [Thread-199 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-17T09:48:28,932 DEBUG [Thread-199 {}] bucket.BucketCache(1584): Cached File Entry:> 2024-11-17T09:48:28,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40129 2024-11-17T09:48:28,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40129 2024-11-17T09:48:28,932 INFO [Thread-199 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-17T09:48:28,932 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40129 2024-11-17T09:48:28,933 INFO [Thread-199 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-17T09:48:28,933 INFO [Thread-199 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence is complete. 2024-11-17T09:48:28,934 INFO [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(746): ClusterId : e242b801-37af-4aed-a76c-156695abeb2f 2024-11-17T09:48:28,934 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-17T09:48:28,936 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-17T09:48:28,936 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-17T09:48:28,939 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-17T09:48:28,940 DEBUG [RS:2;70e37c475432:40129 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650f6c0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=70e37c475432/172.17.0.2:0 2024-11-17T09:48:28,952 DEBUG [RS:2;70e37c475432:40129 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;70e37c475432:40129 2024-11-17T09:48:28,952 INFO [RS:2;70e37c475432:40129 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-17T09:48:28,952 INFO [RS:2;70e37c475432:40129 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-17T09:48:28,952 DEBUG [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-17T09:48:28,953 INFO [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(2659): reportForDuty to master=70e37c475432,40981,1731836905677 with port=40129, startcode=1731836908918 2024-11-17T09:48:28,953 DEBUG [RS:2;70e37c475432:40129 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-17T09:48:28,955 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36569, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-17T09:48:28,956 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40981 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 70e37c475432,40129,1731836908918 2024-11-17T09:48:28,956 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40981 {}] master.ServerManager(517): Registering regionserver=70e37c475432,40129,1731836908918 2024-11-17T09:48:28,957 DEBUG [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8 2024-11-17T09:48:28,957 DEBUG [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-17T09:48:28,957 DEBUG [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-17T09:48:28,959 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:48:28,959 DEBUG [RS:2;70e37c475432:40129 {}] zookeeper.ZKUtil(111): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/70e37c475432,40129,1731836908918 2024-11-17T09:48:28,959 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,959 WARN [RS:2;70e37c475432:40129 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-17T09:48:28,959 INFO [RS:2;70e37c475432:40129 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-17T09:48:28,959 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 21e9bd53ef6be3f74e171f1e8af18ff4, NAME => 'testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.', STARTKEY => '', ENDKEY => ''} 2024-11-17T09:48:28,960 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [70e37c475432,40129,1731836908918] 2024-11-17T09:48:28,960 DEBUG [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,40129,1731836908918 2024-11-17T09:48:28,960 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731836906717 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,960 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-17T09:48:28,960 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,960 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,961 INFO [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,962 INFO [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 21e9bd53ef6be3f74e171f1e8af18ff4 columnFamilyName family 2024-11-17T09:48:28,962 DEBUG [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-17T09:48:28,963 DEBUG [StoreFileOpener-21e9bd53ef6be3f74e171f1e8af18ff4-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c, delay=954 ms 2024-11-17T09:48:28,964 DEBUG [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c 2024-11-17T09:48:28,965 INFO [StoreOpener-21e9bd53ef6be3f74e171f1e8af18ff4-1 {}] regionserver.HStore(327): Store=21e9bd53ef6be3f74e171f1e8af18ff4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-17T09:48:28,965 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,965 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,966 INFO [RS:2;70e37c475432:40129 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-17T09:48:28,966 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,966 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,966 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,967 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,968 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 21e9bd53ef6be3f74e171f1e8af18ff4; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65765747, jitterRate=-0.020014002919197083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-17T09:48:28,968 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:28,968 INFO [RS:2;70e37c475432:40129 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-17T09:48:28,968 INFO [RS:2;70e37c475432:40129 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-17T09:48:28,968 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,969 INFO [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-17T09:48:28,969 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 21e9bd53ef6be3f74e171f1e8af18ff4: Running coprocessor pre-open hook at 1731836908960Writing region info on filesystem at 1731836908960Initializing all the Stores at 1731836908961 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731836908961Cleaning up temporary data from old regions at 1731836908966 (+5 ms)Running coprocessor post-open hooks at 1731836908968 (+2 ms)Region opened successfully at 1731836908969 (+1 ms) 2024-11-17T09:48:28,969 INFO [RS:2;70e37c475432:40129 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-17T09:48:28,969 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,969 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,969 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4., pid=9, masterSystemTime=1731836908957 2024-11-17T09:48:28,969 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,969 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/70e37c475432:0, corePoolSize=2, maxPoolSize=2 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/70e37c475432:0, corePoolSize=1, maxPoolSize=1 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:28,970 DEBUG [RS:2;70e37c475432:40129 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/70e37c475432:0, corePoolSize=3, maxPoolSize=3 2024-11-17T09:48:28,970 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,971 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,971 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,971 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,971 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,971 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40129,1731836908918-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-17T09:48:28,972 DEBUG [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,972 INFO [RS_OPEN_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:28,972 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=21e9bd53ef6be3f74e171f1e8af18ff4, regionState=OPEN, openSeqNum=10, regionLocation=70e37c475432,39601,1731836905727 2024-11-17T09:48:28,975 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 21e9bd53ef6be3f74e171f1e8af18ff4, server=70e37c475432,39601,1731836905727 because future has completed 2024-11-17T09:48:28,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-17T09:48:28,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 21e9bd53ef6be3f74e171f1e8af18ff4, server=70e37c475432,39601,1731836905727 in 171 msec 2024-11-17T09:48:28,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-17T09:48:28,979 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731836906717, region=21e9bd53ef6be3f74e171f1e8af18ff4, ASSIGN in 328 msec 2024-11-17T09:48:28,979 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 70e37c475432,43629,1731836905778 after splitting done 2024-11-17T09:48:28,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 70e37c475432,43629,1731836905778, splitWal=true, meta=false in 344 msec 2024-11-17T09:48:28,987 INFO [RS:2;70e37c475432:40129 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-17T09:48:28,987 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=70e37c475432,40129,1731836908918-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,988 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:28,988 INFO [RS:2;70e37c475432:40129 {}] regionserver.Replication(171): 70e37c475432,40129,1731836908918 started 2024-11-17T09:48:29,002 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-17T09:48:29,002 INFO [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(1482): Serving as 70e37c475432,40129,1731836908918, RpcServer on 70e37c475432/172.17.0.2:40129, sessionid=0x1012553f3470003 2024-11-17T09:48:29,003 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-17T09:48:29,003 DEBUG [RS:2;70e37c475432:40129 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 70e37c475432,40129,1731836908918 2024-11-17T09:48:29,003 DEBUG [RS:2;70e37c475432:40129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,40129,1731836908918' 2024-11-17T09:48:29,003 DEBUG [RS:2;70e37c475432:40129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-17T09:48:29,003 DEBUG [RS:2;70e37c475432:40129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-17T09:48:29,004 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-17T09:48:29,004 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-17T09:48:29,004 DEBUG [RS:2;70e37c475432:40129 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 70e37c475432,40129,1731836908918 2024-11-17T09:48:29,004 DEBUG [RS:2;70e37c475432:40129 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '70e37c475432,40129,1731836908918' 2024-11-17T09:48:29,004 DEBUG [RS:2;70e37c475432:40129 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-17T09:48:29,004 DEBUG [RS:2;70e37c475432:40129 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-17T09:48:29,005 DEBUG [RS:2;70e37c475432:40129 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-17T09:48:29,005 INFO [RS:2;70e37c475432:40129 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-17T09:48:29,005 INFO [RS:2;70e37c475432:40129 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-17T09:48:29,105 INFO [RS:2;70e37c475432:40129 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-17T09:48:29,106 INFO [RS:2;70e37c475432:40129 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=70e37c475432%2C40129%2C1731836908918, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,40129,1731836908918, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs, maxLogs=55 2024-11-17T09:48:29,124 INFO [RS:2;70e37c475432:40129 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/WALs/70e37c475432,40129,1731836908918/70e37c475432%2C40129%2C1731836908918.1731836909107 2024-11-17T09:48:29,124 DEBUG [RS:2;70e37c475432:40129 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-17T09:48:29,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-17T09:48:29,503 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:48:29,503 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:29,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:29,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:29,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-17T09:48:29,503 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-17T09:48:29,503 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1226522458, stopped=false 2024-11-17T09:48:29,504 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=70e37c475432,40981,1731836905677 2024-11-17T09:48:29,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:29,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:29,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-17T09:48:29,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:29,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:29,505 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T09:48:29,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:29,506 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-17T09:48:29,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:29,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:29,506 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:29,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:29,507 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '70e37c475432,39601,1731836905727' ***** 2024-11-17T09:48:29,507 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T09:48:29,507 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '70e37c475432,40129,1731836908918' ***** 2024-11-17T09:48:29,507 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-17T09:48:29,507 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-17T09:48:29,507 INFO [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T09:48:29,507 INFO [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1738): Disabling cache 2024-11-17T09:48:29,507 INFO [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence 2024-11-17T09:48:29,507 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T09:48:29,507 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T09:48:29,507 INFO [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e1d9c50b-8c51-bb62-2ad7-1746108f50b0/bucket.persistence 2024-11-17T09:48:29,507 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-17T09:48:29,507 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T09:48:29,507 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-17T09:48:29,507 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-17T09:48:29,508 INFO [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T09:48:29,508 INFO [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-17T09:48:29,508 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T09:48:29,508 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-17T09:48:29,518 DEBUG [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1487): Thread RS:0;70e37c475432:39601 started persisting bucket cache to file 2024-11-17T09:48:29,518 DEBUG [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:29,518 DEBUG [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-17T09:48:29,518 DEBUG [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1487): Thread RS:2;70e37c475432:40129 started persisting bucket cache to file 2024-11-17T09:48:29,518 DEBUG [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-17T09:48:29,518 DEBUG [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-17T09:48:29,521 DEBUG [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-17T09:48:29,521 DEBUG [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731836909518 2024-11-17T09:48:29,521 DEBUG [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-17T09:48:29,521 DEBUG [RS:0;70e37c475432:39601 {}] bucket.BucketCache(1509): Thread RS:0;70e37c475432:39601 finished persisting bucket cache to file, renaming 2024-11-17T09:48:29,521 DEBUG [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731836909518 2024-11-17T09:48:29,522 DEBUG [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1509): Thread RS:2;70e37c475432:40129 finished persisting bucket cache to file, renaming 2024-11-17T09:48:29,522 WARN [RS:2;70e37c475432:40129 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-17T09:48:29,522 INFO [RS:0;70e37c475432:39601 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T09:48:29,522 INFO [RS:2;70e37c475432:40129 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-17T09:48:29,522 INFO [RS:0;70e37c475432:39601 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T09:48:29,522 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T09:48:29,522 INFO [RS:2;70e37c475432:40129 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-17T09:48:29,522 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-17T09:48:29,522 INFO [RS:0;70e37c475432:39601 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T09:48:29,522 INFO [RS:2;70e37c475432:40129 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-17T09:48:29,522 INFO [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(959): stopping server 70e37c475432,40129,1731836908918 2024-11-17T09:48:29,522 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(3091): Received CLOSE for 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:29,522 INFO [RS:2;70e37c475432:40129 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:48:29,522 INFO [RS:2;70e37c475432:40129 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;70e37c475432:40129. 2024-11-17T09:48:29,522 DEBUG [RS:2;70e37c475432:40129 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:29,522 DEBUG [RS:2;70e37c475432:40129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:29,522 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(959): stopping server 70e37c475432,39601,1731836905727 2024-11-17T09:48:29,522 INFO [RS:0;70e37c475432:39601 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:48:29,522 INFO [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(976): stopping server 70e37c475432,40129,1731836908918; all regions closed. 2024-11-17T09:48:29,522 INFO [RS:0;70e37c475432:39601 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;70e37c475432:39601. 2024-11-17T09:48:29,522 DEBUG [RS:0;70e37c475432:39601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-17T09:48:29,522 DEBUG [RS:0;70e37c475432:39601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:29,522 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T09:48:29,523 INFO [RS:0;70e37c475432:39601 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T09:48:29,523 INFO [RS:0;70e37c475432:39601 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T09:48:29,523 INFO [RS:0;70e37c475432:39601 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 21e9bd53ef6be3f74e171f1e8af18ff4, disabling compactions & flushes 2024-11-17T09:48:29,523 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-17T09:48:29,523 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. after waiting 0 ms 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:29,523 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-17T09:48:29,523 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 21e9bd53ef6be3f74e171f1e8af18ff4=testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4.} 2024-11-17T09:48:29,523 DEBUG [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 21e9bd53ef6be3f74e171f1e8af18ff4 2024-11-17T09:48:29,523 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-17T09:48:29,523 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-17T09:48:29,523 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-17T09:48:29,523 DEBUG [StoreFileCloser-21e9bd53ef6be3f74e171f1e8af18ff4-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/family/ee52269334d549a1a1d9febddb7ff78c 2024-11-17T09:48:29,524 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-17T09:48:29,524 DEBUG [RS:2;70e37c475432:40129 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs 2024-11-17T09:48:29,524 INFO [RS:2;70e37c475432:40129 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C40129%2C1731836908918:(num 1731836909107) 2024-11-17T09:48:29,524 DEBUG [RS:2;70e37c475432:40129 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:29,524 INFO [RS:2;70e37c475432:40129 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:29,525 INFO [RS:2;70e37c475432:40129 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:48:29,525 INFO [RS:2;70e37c475432:40129 {}] hbase.ChoreService(370): Chore service for: regionserver/70e37c475432:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-17T09:48:29,525 INFO [RS:2;70e37c475432:40129 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-17T09:48:29,525 INFO [RS:2;70e37c475432:40129 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-17T09:48:29,525 INFO [regionserver/70e37c475432:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:48:29,525 INFO [RS:2;70e37c475432:40129 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-17T09:48:29,525 INFO [RS:2;70e37c475432:40129 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:48:29,525 INFO [RS:2;70e37c475432:40129 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40129 2024-11-17T09:48:29,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/70e37c475432,40129,1731836908918 2024-11-17T09:48:29,529 INFO [RS:2;70e37c475432:40129 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:48:29,529 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:48:29,530 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/default/testBlockEvictionOnGracefulStop1731836906717/21e9bd53ef6be3f74e171f1e8af18ff4/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-17T09:48:29,530 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [70e37c475432,40129,1731836908918] 2024-11-17T09:48:29,530 INFO [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:29,530 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 21e9bd53ef6be3f74e171f1e8af18ff4: Waiting for close lock at 1731836909522Running coprocessor pre-close hooks at 1731836909523 (+1 ms)Disabling compacts and flushes for region at 1731836909523Disabling writes for close at 1731836909523Writing region close event to WAL at 1731836909524 (+1 ms)Running coprocessor post-close hooks at 1731836909530 (+6 ms)Closed at 1731836909530 2024-11-17T09:48:29,531 DEBUG [RS_CLOSE_REGION-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4. 2024-11-17T09:48:29,531 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/70e37c475432,40129,1731836908918 already deleted, retry=false 2024-11-17T09:48:29,531 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 70e37c475432,40129,1731836908918 expired; onlineServers=1 2024-11-17T09:48:29,535 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4d69d60f80a24cb9b251654ba3955caa 2024-11-17T09:48:29,550 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/info/4d69d60f80a24cb9b251654ba3955caa is 209, key is testBlockEvictionOnGracefulStop1731836906717,,1731836906721.21e9bd53ef6be3f74e171f1e8af18ff4./info:regioninfo/1731836908972/Put/seqid=0 2024-11-17T09:48:29,551 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/info/4d69d60f80a24cb9b251654ba3955caa 2024-11-17T09:48:29,556 INFO [regionserver/70e37c475432:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:29,560 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 906d985e22464afca1f052f4349f6a10 2024-11-17T09:48:29,572 INFO [regionserver/70e37c475432:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:29,573 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/ns/906d985e22464afca1f052f4349f6a10 is 43, key is default/ns:d/1731836906688/Put/seqid=0 2024-11-17T09:48:29,574 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/ns/906d985e22464afca1f052f4349f6a10 2024-11-17T09:48:29,583 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b65f30f540244cba9c38d6bf360c741f 2024-11-17T09:48:29,598 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/table/b65f30f540244cba9c38d6bf360c741f is 80, key is testBlockEvictionOnGracefulStop1731836906717/table:state/1731836907096/Put/seqid=0 2024-11-17T09:48:29,599 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/table/b65f30f540244cba9c38d6bf360c741f 2024-11-17T09:48:29,601 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/info/4d69d60f80a24cb9b251654ba3955caa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/4d69d60f80a24cb9b251654ba3955caa 2024-11-17T09:48:29,602 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/4d69d60f80a24cb9b251654ba3955caa, delay=983 ms 2024-11-17T09:48:29,604 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/4d69d60f80a24cb9b251654ba3955caa, entries=14, sequenceid=13, filesize=7.5 K 2024-11-17T09:48:29,604 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/ns/906d985e22464afca1f052f4349f6a10 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/906d985e22464afca1f052f4349f6a10 2024-11-17T09:48:29,606 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/906d985e22464afca1f052f4349f6a10, delay=974 ms 2024-11-17T09:48:29,607 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/906d985e22464afca1f052f4349f6a10, entries=2, sequenceid=13, filesize=5.0 K 2024-11-17T09:48:29,608 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/.tmp/table/b65f30f540244cba9c38d6bf360c741f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/b65f30f540244cba9c38d6bf360c741f 2024-11-17T09:48:29,609 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/b65f30f540244cba9c38d6bf360c741f, delay=982 ms 2024-11-17T09:48:29,610 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/b65f30f540244cba9c38d6bf360c741f, entries=2, sequenceid=13, filesize=5.3 K 2024-11-17T09:48:29,611 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=13, compaction requested=false 2024-11-17T09:48:29,612 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/info/4d69d60f80a24cb9b251654ba3955caa 2024-11-17T09:48:29,612 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/ns/906d985e22464afca1f052f4349f6a10 2024-11-17T09:48:29,613 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/table/b65f30f540244cba9c38d6bf360c741f 2024-11-17T09:48:29,617 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-17T09:48:29,617 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-17T09:48:29,618 INFO [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-17T09:48:29,618 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731836909523Running coprocessor pre-close hooks at 1731836909523Disabling compacts and flushes for region at 1731836909523Disabling writes for close at 1731836909523Obtaining lock to block concurrent updates at 1731836909524 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731836909524Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731836909524Flushing stores of hbase:meta,,1.1588230740 at 1731836909524Flushing 1588230740/info: creating writer at 1731836909524Flushing 1588230740/info: appending metadata at 1731836909549 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731836909549Flushing 1588230740/ns: creating writer at 1731836909553 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731836909573 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731836909573Flushing 1588230740/table: creating writer at 1731836909576 (+3 ms)Flushing 1588230740/table: appending metadata at 1731836909597 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731836909597Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35fbb354: reopening flushed file at 1731836909601 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37e3b56d: reopening flushed file at 1731836909604 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e4474cb: reopening flushed file at 1731836909607 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=13, compaction requested=false at 1731836909611 (+4 ms)Writing region close event to WAL at 1731836909613 (+2 ms)Running coprocessor post-close hooks at 1731836909617 (+4 ms)Closed at 1731836909618 (+1 ms) 2024-11-17T09:48:29,618 DEBUG [RS_CLOSE_META-regionserver/70e37c475432:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-17T09:48:29,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:29,630 INFO [RS:2;70e37c475432:40129 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:48:29,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40129-0x1012553f3470003, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:29,630 INFO [RS:2;70e37c475432:40129 {}] regionserver.HRegionServer(1031): Exiting; stopping=70e37c475432,40129,1731836908918; zookeeper connection closed. 2024-11-17T09:48:29,631 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@169acfbb {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@169acfbb 2024-11-17T09:48:29,723 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(976): stopping server 70e37c475432,39601,1731836905727; all regions closed. 2024-11-17T09:48:29,725 DEBUG [RS:0;70e37c475432:39601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs 2024-11-17T09:48:29,725 INFO [RS:0;70e37c475432:39601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C39601%2C1731836905727.meta:.meta(num 1731836906641) 2024-11-17T09:48:29,727 DEBUG [RS:0;70e37c475432:39601 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/oldWALs 2024-11-17T09:48:29,727 INFO [RS:0;70e37c475432:39601 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 70e37c475432%2C39601%2C1731836905727:(num 1731836906296) 2024-11-17T09:48:29,727 DEBUG [RS:0;70e37c475432:39601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-17T09:48:29,727 INFO [RS:0;70e37c475432:39601 {}] regionserver.LeaseManager(133): Closed leases 2024-11-17T09:48:29,727 INFO [RS:0;70e37c475432:39601 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:48:29,728 INFO [RS:0;70e37c475432:39601 {}] hbase.ChoreService(370): Chore service for: regionserver/70e37c475432:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-17T09:48:29,728 INFO [RS:0;70e37c475432:39601 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:48:29,728 INFO [regionserver/70e37c475432:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:48:29,728 INFO [RS:0;70e37c475432:39601 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39601 2024-11-17T09:48:29,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-17T09:48:29,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/70e37c475432,39601,1731836905727 2024-11-17T09:48:29,730 INFO [RS:0;70e37c475432:39601 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:48:29,732 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [70e37c475432,39601,1731836905727] 2024-11-17T09:48:29,734 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/70e37c475432,39601,1731836905727 already deleted, retry=false 2024-11-17T09:48:29,734 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 70e37c475432,39601,1731836905727 expired; onlineServers=0 2024-11-17T09:48:29,734 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '70e37c475432,40981,1731836905677' ***** 2024-11-17T09:48:29,734 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-17T09:48:29,734 INFO [M:0;70e37c475432:40981 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-17T09:48:29,734 INFO [M:0;70e37c475432:40981 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-17T09:48:29,734 DEBUG [M:0;70e37c475432:40981 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-17T09:48:29,734 DEBUG [M:0;70e37c475432:40981 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-17T09:48:29,734 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-17T09:48:29,734 DEBUG [master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836906032 {}] cleaner.HFileCleaner(306): Exit Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.large.0-1731836906032,5,FailOnTimeoutGroup] 2024-11-17T09:48:29,734 DEBUG [master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836906032 {}] cleaner.HFileCleaner(306): Exit Thread[master/70e37c475432:0:becomeActiveMaster-HFileCleaner.small.0-1731836906032,5,FailOnTimeoutGroup] 2024-11-17T09:48:29,734 INFO [M:0;70e37c475432:40981 {}] hbase.ChoreService(370): Chore service for: master/70e37c475432:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-17T09:48:29,735 INFO [M:0;70e37c475432:40981 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-17T09:48:29,735 DEBUG [M:0;70e37c475432:40981 {}] master.HMaster(1795): Stopping service threads 2024-11-17T09:48:29,735 INFO [M:0;70e37c475432:40981 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-17T09:48:29,735 INFO [M:0;70e37c475432:40981 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-17T09:48:29,735 INFO [M:0;70e37c475432:40981 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-17T09:48:29,735 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-17T09:48:29,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-17T09:48:29,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-17T09:48:29,736 DEBUG [M:0;70e37c475432:40981 {}] zookeeper.ZKUtil(347): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-17T09:48:29,736 WARN [M:0;70e37c475432:40981 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-17T09:48:29,736 INFO [M:0;70e37c475432:40981 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/.lastflushedseqids 2024-11-17T09:48:29,743 INFO [M:0;70e37c475432:40981 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-17T09:48:29,743 INFO [M:0;70e37c475432:40981 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-17T09:48:29,743 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-17T09:48:29,743 INFO [M:0;70e37c475432:40981 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:29,743 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:29,744 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-17T09:48:29,744 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:29,744 INFO [M:0;70e37c475432:40981 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-17T09:48:29,754 DEBUG [M:0;70e37c475432:40981 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2302da870bca44bab1cc063a4a647843 2024-11-17T09:48:29,770 DEBUG [M:0;70e37c475432:40981 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2302da870bca44bab1cc063a4a647843 is 82, key is hbase:meta,,1/info:regioninfo/1731836906673/Put/seqid=0 2024-11-17T09:48:29,770 INFO [M:0;70e37c475432:40981 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2302da870bca44bab1cc063a4a647843 2024-11-17T09:48:29,778 DEBUG [M:0;70e37c475432:40981 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ea65531ea28d44e994437e4aea9b1ccd 2024-11-17T09:48:29,791 DEBUG [M:0;70e37c475432:40981 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea65531ea28d44e994437e4aea9b1ccd is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731836907100/Put/seqid=0 2024-11-17T09:48:29,792 INFO [M:0;70e37c475432:40981 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea65531ea28d44e994437e4aea9b1ccd 2024-11-17T09:48:29,799 DEBUG [M:0;70e37c475432:40981 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5e4401cf7e8e46c18c87eb05c5f4f46d 2024-11-17T09:48:29,813 DEBUG [M:0;70e37c475432:40981 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e4401cf7e8e46c18c87eb05c5f4f46d is 69, key is 70e37c475432,39601,1731836905727/rs:state/1731836906129/Put/seqid=0 2024-11-17T09:48:29,813 INFO [M:0;70e37c475432:40981 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e4401cf7e8e46c18c87eb05c5f4f46d 2024-11-17T09:48:29,815 INFO [M:0;70e37c475432:40981 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5e4401cf7e8e46c18c87eb05c5f4f46d 2024-11-17T09:48:29,816 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2302da870bca44bab1cc063a4a647843 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2302da870bca44bab1cc063a4a647843 2024-11-17T09:48:29,817 INFO [M:0;70e37c475432:40981 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2302da870bca44bab1cc063a4a647843, entries=8, sequenceid=83, filesize=5.5 K 2024-11-17T09:48:29,818 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ea65531ea28d44e994437e4aea9b1ccd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ea65531ea28d44e994437e4aea9b1ccd 2024-11-17T09:48:29,819 INFO [M:0;70e37c475432:40981 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ea65531ea28d44e994437e4aea9b1ccd, entries=9, sequenceid=83, filesize=6.4 K 2024-11-17T09:48:29,819 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/5e4401cf7e8e46c18c87eb05c5f4f46d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e4401cf7e8e46c18c87eb05c5f4f46d 2024-11-17T09:48:29,821 INFO [M:0;70e37c475432:40981 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5e4401cf7e8e46c18c87eb05c5f4f46d 2024-11-17T09:48:29,821 INFO [M:0;70e37c475432:40981 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d44e71bf-53b7-8713-8c70-83bdd484cce3/4ecc859a-fb36-53c8-29e3-84d2eee5a2e8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/5e4401cf7e8e46c18c87eb05c5f4f46d, entries=3, sequenceid=83, filesize=5.3 K 2024-11-17T09:48:29,822 INFO [M:0;70e37c475432:40981 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false 2024-11-17T09:48:29,823 INFO [M:0;70e37c475432:40981 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-17T09:48:29,823 DEBUG [M:0;70e37c475432:40981 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731836909743Disabling compacts and flushes for region at 1731836909743Disabling writes for close at 1731836909744 (+1 ms)Obtaining lock to block concurrent updates at 1731836909744Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731836909744Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1731836909744Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731836909745 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731836909745Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731836909769 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731836909769Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731836909772 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731836909791 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731836909791Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731836909793 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731836909812 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731836909812Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3417850d: reopening flushed file at 1731836909815 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@477cebcc: reopening flushed file at 1731836909817 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b2709c1: reopening flushed file at 1731836909819 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false at 1731836909822 (+3 ms)Writing region close event to WAL at 1731836909823 (+1 ms)Closed at 1731836909823 2024-11-17T09:48:29,825 INFO [M:0;70e37c475432:40981 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-17T09:48:29,825 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-17T09:48:29,825 INFO [M:0;70e37c475432:40981 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40981 2024-11-17T09:48:29,825 INFO [M:0;70e37c475432:40981 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-17T09:48:29,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:29,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39601-0x1012553f3470001, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:29,833 INFO [RS:0;70e37c475432:39601 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:48:29,833 INFO [RS:0;70e37c475432:39601 {}] regionserver.HRegionServer(1031): Exiting; stopping=70e37c475432,39601,1731836905727; zookeeper connection closed. 2024-11-17T09:48:29,833 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@52fea24c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@52fea24c 2024-11-17T09:48:29,833 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-17T09:48:29,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:29,927 INFO [M:0;70e37c475432:40981 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-17T09:48:29,927 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40981-0x1012553f3470000, quorum=127.0.0.1:56799, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-17T09:48:29,930 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-17T09:48:29,931 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-17T09:48:29,931 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-17T09:48:29,938 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 50) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1085 (was 1171), ProcessCount=11 (was 11), AvailableMemoryMB=3512 (was 3578)