2024-11-10 05:49:25,408 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-10 05:49:25,429 main DEBUG Took 0.018185 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 05:49:25,429 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 05:49:25,430 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 05:49:25,431 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 05:49:25,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,443 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 05:49:25,461 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,463 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,464 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,464 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,465 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,466 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,467 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,467 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,468 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,468 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,470 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,470 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,471 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,471 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,472 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,472 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,473 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,473 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,474 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,474 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,475 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,476 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,476 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,477 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:49:25,477 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,478 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 05:49:25,480 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:49:25,481 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 05:49:25,484 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 05:49:25,484 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 05:49:25,486 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 05:49:25,486 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 05:49:25,498 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 05:49:25,501 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 05:49:25,503 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 05:49:25,504 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 05:49:25,504 main DEBUG createAppenders(={Console}) 2024-11-10 05:49:25,505 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-10 05:49:25,506 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-10 05:49:25,506 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-10 05:49:25,507 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 05:49:25,507 main DEBUG OutputStream closed 2024-11-10 05:49:25,508 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 05:49:25,508 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 05:49:25,509 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-10 05:49:25,588 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 05:49:25,590 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 05:49:25,591 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 05:49:25,592 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 05:49:25,593 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 05:49:25,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 05:49:25,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 05:49:25,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 05:49:25,594 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 05:49:25,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 05:49:25,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 05:49:25,595 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 05:49:25,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 05:49:25,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 05:49:25,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 05:49:25,596 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 05:49:25,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 05:49:25,597 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 05:49:25,600 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 05:49:25,600 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-10 05:49:25,600 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 05:49:25,601 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-10T05:49:25,616 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-10 05:49:25,619 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 05:49:25,619 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T05:49:25,896 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80 2024-11-10T05:49:25,924 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=8683 2024-11-10T05:49:26,040 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T05:49:26,134 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:26,149 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/cluster_fab6fc59-9ca6-2555-aaf8-36e8ac04f455, deleteOnExit=true 2024-11-10T05:49:26,239 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/cluster_fab6fc59-9ca6-2555-aaf8-36e8ac04f455/zookeeper_0, clientPort=57491, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/cluster_fab6fc59-9ca6-2555-aaf8-36e8ac04f455/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/cluster_fab6fc59-9ca6-2555-aaf8-36e8ac04f455/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T05:49:26,252 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57491 2024-11-10T05:49:26,252 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:26,348 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd with version=8 2024-11-10T05:49:26,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/hbase-staging 2024-11-10T05:49:26,450 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T05:49:26,744 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4999977c7e1b:0 server-side Connection retries=45 2024-11-10T05:49:26,756 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:26,757 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:26,769 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T05:49:26,769 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:26,770 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T05:49:26,925 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T05:49:27,018 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T05:49:27,057 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T05:49:27,068 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T05:49:27,072 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T05:49:27,116 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12908 (auto-detected) 2024-11-10T05:49:27,117 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-10T05:49:27,137 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:43297 2024-11-10T05:49:27,160 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43297 connecting to ZooKeeper ensemble=127.0.0.1:57491 2024-11-10T05:49:27,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:432970x0, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T05:49:27,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43297-0x10190ba2cb30000 connected 2024-11-10T05:49:27,225 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:27,226 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:27,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:27,237 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd, hbase.cluster.distributed=false 2024-11-10T05:49:27,263 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T05:49:27,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43297 2024-11-10T05:49:27,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43297 2024-11-10T05:49:27,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43297 2024-11-10T05:49:27,270 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43297 2024-11-10T05:49:27,271 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43297 2024-11-10T05:49:27,358 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4999977c7e1b:0 server-side Connection retries=45 2024-11-10T05:49:27,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:27,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:27,360 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T05:49:27,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:27,361 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T05:49:27,364 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T05:49:27,366 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T05:49:27,367 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:32937 2024-11-10T05:49:27,369 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:32937 connecting to ZooKeeper ensemble=127.0.0.1:57491 2024-11-10T05:49:27,369 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:27,370 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:27,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:329370x0, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T05:49:27,375 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:32937-0x10190ba2cb30001 connected 2024-11-10T05:49:27,375 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:27,381 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T05:49:27,393 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.cache 2024-11-10T05:49:27,399 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T05:49:27,400 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T05:49:27,402 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T05:49:27,404 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T05:49:27,404 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T05:49:27,411 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T05:49:27,413 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T05:49:27,413 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.persistence is complete. 2024-11-10T05:49:27,418 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T05:49:27,420 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T05:49:27,427 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T05:49:27,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=32937 2024-11-10T05:49:27,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=32937 2024-11-10T05:49:27,430 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=32937 2024-11-10T05:49:27,433 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=32937 2024-11-10T05:49:27,436 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=32937 2024-11-10T05:49:27,455 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4999977c7e1b:0 server-side Connection retries=45 2024-11-10T05:49:27,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:27,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:27,456 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T05:49:27,457 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:27,457 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T05:49:27,457 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T05:49:27,458 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T05:49:27,459 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35809 2024-11-10T05:49:27,462 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35809 connecting to ZooKeeper ensemble=127.0.0.1:57491 2024-11-10T05:49:27,462 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:27,463 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:27,467 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:358090x0, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T05:49:27,468 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35809-0x10190ba2cb30002 connected 2024-11-10T05:49:27,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:27,470 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T05:49:27,471 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.cache 2024-11-10T05:49:27,471 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T05:49:27,472 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T05:49:27,472 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T05:49:27,474 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T05:49:27,474 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T05:49:27,474 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T05:49:27,475 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T05:49:27,475 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.persistence is complete. 2024-11-10T05:49:27,475 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T05:49:27,476 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T05:49:27,478 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T05:49:27,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35809 2024-11-10T05:49:27,484 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35809 2024-11-10T05:49:27,485 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35809 2024-11-10T05:49:27,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35809 2024-11-10T05:49:27,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35809 2024-11-10T05:49:27,503 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4999977c7e1b:43297 2024-11-10T05:49:27,504 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4999977c7e1b,43297,1731217766535 2024-11-10T05:49:27,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:27,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:27,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:27,515 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4999977c7e1b,43297,1731217766535 2024-11-10T05:49:27,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T05:49:27,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T05:49:27,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:27,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:27,542 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:27,543 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T05:49:27,545 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4999977c7e1b,43297,1731217766535 from backup master directory 2024-11-10T05:49:27,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:27,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4999977c7e1b,43297,1731217766535 2024-11-10T05:49:27,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:27,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:27,549 WARN [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T05:49:27,550 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4999977c7e1b,43297,1731217766535 2024-11-10T05:49:27,552 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-10T05:49:27,554 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-10T05:49:27,562 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/hbase.id] with ID: 4d71b06b-660c-45bd-a67f-0b1b07ca6d29 2024-11-10T05:49:27,562 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.tmp/hbase.id 2024-11-10T05:49:27,568 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/hbase.id] 2024-11-10T05:49:27,616 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:27,622 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T05:49:27,643 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-11-10T05:49:27,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:27,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:27,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:27,679 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:49:27,683 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:49:27,732 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T05:49:27,740 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:27,803 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store 2024-11-10T05:49:27,828 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T05:49:27,833 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:27,834 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:49:27,835 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:27,835 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:27,837 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:49:27,838 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:27,838 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:27,839 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731217767834Disabling compacts and flushes for region at 1731217767834Disabling writes for close at 1731217767837 (+3 ms)Writing region close event to WAL at 1731217767838 (+1 ms)Closed at 1731217767838 2024-11-10T05:49:27,839 WARN [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/.initializing 2024-11-10T05:49:27,840 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/WALs/4999977c7e1b,43297,1731217766535 2024-11-10T05:49:27,854 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:49:27,869 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4999977c7e1b%2C43297%2C1731217766535, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/WALs/4999977c7e1b,43297,1731217766535, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/oldWALs, maxLogs=10 2024-11-10T05:49:27,909 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/WALs/4999977c7e1b,43297,1731217766535/4999977c7e1b%2C43297%2C1731217766535.1731217767875 2024-11-10T05:49:27,910 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:27,911 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:27,911 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:27,915 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:27,916 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:27,962 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:27,993 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:49:27,997 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:27,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:27,999 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,005 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:49:28,006 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,006 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:28,007 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,011 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:49:28,012 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,012 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:28,013 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:49:28,018 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:28,019 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,023 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,023 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,026 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,027 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,029 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:49:28,032 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:28,039 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:49:28,039 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67222523, jitterRate=0.0016936510801315308}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:49:28,047 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731217767929Initializing all the Stores at 1731217767929Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217767930 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217767931 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217767931Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217767931Cleaning up temporary data from old regions at 1731217768027 (+96 ms)Region opened successfully at 1731217768046 (+19 ms) 2024-11-10T05:49:28,048 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:49:28,084 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c34e25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4999977c7e1b/172.17.0.2:0 2024-11-10T05:49:28,120 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T05:49:28,134 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T05:49:28,134 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:49:28,137 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T05:49:28,138 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T05:49:28,144 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-10T05:49:28,145 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:49:28,172 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T05:49:28,182 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T05:49:28,185 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T05:49:28,187 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T05:49:28,189 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T05:49:28,191 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T05:49:28,193 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T05:49:28,197 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T05:49:28,198 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T05:49:28,200 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T05:49:28,201 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T05:49:28,222 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T05:49:28,224 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T05:49:28,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:28,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:28,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:28,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,238 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,239 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,242 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4999977c7e1b,43297,1731217766535, sessionid=0x10190ba2cb30000, setting cluster-up flag (Was=false) 2024-11-10T05:49:28,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,255 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,261 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T05:49:28,263 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4999977c7e1b,43297,1731217766535 2024-11-10T05:49:28,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,273 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T05:49:28,275 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4999977c7e1b,43297,1731217766535 2024-11-10T05:49:28,281 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T05:49:28,290 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(746): ClusterId : 4d71b06b-660c-45bd-a67f-0b1b07ca6d29 2024-11-10T05:49:28,290 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(746): ClusterId : 4d71b06b-660c-45bd-a67f-0b1b07ca6d29 2024-11-10T05:49:28,293 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T05:49:28,293 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T05:49:28,299 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T05:49:28,299 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T05:49:28,299 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T05:49:28,299 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T05:49:28,303 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T05:49:28,303 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T05:49:28,303 DEBUG [RS:0;4999977c7e1b:32937 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee08817, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4999977c7e1b/172.17.0.2:0 2024-11-10T05:49:28,304 DEBUG [RS:1;4999977c7e1b:35809 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba68264, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4999977c7e1b/172.17.0.2:0 2024-11-10T05:49:28,319 DEBUG [RS:0;4999977c7e1b:32937 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4999977c7e1b:32937 2024-11-10T05:49:28,321 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4999977c7e1b:35809 2024-11-10T05:49:28,322 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T05:49:28,322 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T05:49:28,323 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T05:49:28,323 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T05:49:28,323 DEBUG [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T05:49:28,323 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T05:49:28,326 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(2659): reportForDuty to master=4999977c7e1b,43297,1731217766535 with port=35809, startcode=1731217767455 2024-11-10T05:49:28,326 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(2659): reportForDuty to master=4999977c7e1b,43297,1731217766535 with port=32937, startcode=1731217767318 2024-11-10T05:49:28,341 DEBUG [RS:0;4999977c7e1b:32937 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T05:49:28,341 DEBUG [RS:1;4999977c7e1b:35809 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T05:49:28,365 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:28,377 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T05:49:28,385 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T05:49:28,391 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4999977c7e1b,43297,1731217766535 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T05:49:28,400 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:28,400 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:28,401 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:28,401 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:28,401 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4999977c7e1b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T05:49:28,402 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,402 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4999977c7e1b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T05:49:28,402 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,405 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731217798405 2024-11-10T05:49:28,407 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T05:49:28,408 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:28,408 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T05:49:28,409 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T05:49:28,413 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T05:49:28,413 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T05:49:28,414 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T05:49:28,414 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T05:49:28,414 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,414 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T05:49:28,417 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,424 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T05:49:28,427 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T05:49:28,427 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32903, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T05:49:28,427 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45785, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T05:49:28,428 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T05:49:28,432 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T05:49:28,432 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T05:49:28,436 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.large.0-1731217768434,5,FailOnTimeoutGroup] 2024-11-10T05:49:28,437 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.small.0-1731217768436,5,FailOnTimeoutGroup] 2024-11-10T05:49:28,437 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,437 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T05:49:28,438 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43297 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-10T05:49:28,439 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,439 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,439 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T05:49:28,440 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:28,447 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43297 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4999977c7e1b,35809,1731217767455 2024-11-10T05:49:28,451 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43297 {}] master.ServerManager(517): Registering regionserver=4999977c7e1b,35809,1731217767455 2024-11-10T05:49:28,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:28,456 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T05:49:28,461 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:28,461 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T05:49:28,461 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T05:49:28,461 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T05:49:28,462 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:28,462 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T05:49:28,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:28,467 DEBUG [RS:1;4999977c7e1b:35809 {}] zookeeper.ZKUtil(111): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4999977c7e1b,35809,1731217767455 2024-11-10T05:49:28,468 WARN [RS:1;4999977c7e1b:35809 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T05:49:28,468 INFO [RS:1;4999977c7e1b:35809 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:28,468 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,35809,1731217767455 2024-11-10T05:49:28,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4999977c7e1b,35809,1731217767455] 2024-11-10T05:49:28,474 DEBUG [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-10T05:49:28,474 WARN [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-10T05:49:28,478 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T05:49:28,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:28,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T05:49:28,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T05:49:28,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:28,491 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T05:49:28,499 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T05:49:28,501 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T05:49:28,501 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:28,501 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:28,502 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T05:49:28,502 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:28,502 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:28,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T05:49:28,503 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T05:49:28,503 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:49:28,504 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T05:49:28,515 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:49:28,515 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65526499, jitterRate=-0.023579075932502747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:49:28,516 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T05:49:28,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731217768455Initializing all the Stores at 1731217768455Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217768455Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217768455Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217768455Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217768456 (+1 ms)Cleaning up temporary data from old regions at 1731217768503 (+47 ms)Region opened successfully at 1731217768520 (+17 ms) 2024-11-10T05:49:28,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T05:49:28,520 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T05:49:28,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T05:49:28,520 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T05:49:28,521 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T05:49:28,522 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T05:49:28,522 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731217768520Disabling compacts and flushes for region at 1731217768520Disabling writes for close at 1731217768520Writing region close event to WAL at 1731217768522 (+2 ms)Closed at 1731217768522 2024-11-10T05:49:28,522 INFO [RS:1;4999977c7e1b:35809 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T05:49:28,523 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,524 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T05:49:28,525 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:28,525 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T05:49:28,530 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T05:49:28,532 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,533 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,533 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,533 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,533 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,534 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,534 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4999977c7e1b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T05:49:28,534 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,534 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,535 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,535 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,535 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T05:49:28,535 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,535 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,536 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:28,536 DEBUG [RS:1;4999977c7e1b:35809 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:28,537 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,537 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,537 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,538 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,538 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,538 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,35809,1731217767455-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T05:49:28,546 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T05:49:28,548 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T05:49:28,563 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T05:49:28,565 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,35809,1731217767455-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,566 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,566 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.Replication(171): 4999977c7e1b,35809,1731217767455 started 2024-11-10T05:49:28,576 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(2659): reportForDuty to master=4999977c7e1b,43297,1731217766535 with port=32937, startcode=1731217767318 2024-11-10T05:49:28,578 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43297 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4999977c7e1b,32937,1731217767318 2024-11-10T05:49:28,579 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43297 {}] master.ServerManager(517): Registering regionserver=4999977c7e1b,32937,1731217767318 2024-11-10T05:49:28,582 DEBUG [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:28,582 DEBUG [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T05:49:28,582 DEBUG [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T05:49:28,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:28,588 DEBUG [RS:0;4999977c7e1b:32937 {}] zookeeper.ZKUtil(111): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4999977c7e1b,32937,1731217767318 2024-11-10T05:49:28,588 WARN [RS:0;4999977c7e1b:32937 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T05:49:28,588 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4999977c7e1b,32937,1731217767318] 2024-11-10T05:49:28,588 INFO [RS:0;4999977c7e1b:32937 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:28,589 DEBUG [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,32937,1731217767318 2024-11-10T05:49:28,590 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,590 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1482): Serving as 4999977c7e1b,35809,1731217767455, RpcServer on 4999977c7e1b/172.17.0.2:35809, sessionid=0x10190ba2cb30002 2024-11-10T05:49:28,591 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T05:49:28,591 DEBUG [RS:1;4999977c7e1b:35809 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4999977c7e1b,35809,1731217767455 2024-11-10T05:49:28,592 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,35809,1731217767455' 2024-11-10T05:49:28,592 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T05:49:28,593 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T05:49:28,594 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T05:49:28,594 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T05:49:28,595 DEBUG [RS:1;4999977c7e1b:35809 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4999977c7e1b,35809,1731217767455 2024-11-10T05:49:28,595 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,35809,1731217767455' 2024-11-10T05:49:28,595 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T05:49:28,596 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T05:49:28,596 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T05:49:28,596 DEBUG [RS:1;4999977c7e1b:35809 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T05:49:28,597 INFO [RS:1;4999977c7e1b:35809 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T05:49:28,597 INFO [RS:1;4999977c7e1b:35809 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T05:49:28,600 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T05:49:28,600 INFO [RS:0;4999977c7e1b:32937 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T05:49:28,601 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,601 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T05:49:28,602 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T05:49:28,602 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,602 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,602 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,602 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,602 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4999977c7e1b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,603 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:28,604 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:28,604 DEBUG [RS:0;4999977c7e1b:32937 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:28,604 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,605 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,605 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,605 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,605 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,605 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,32937,1731217767318-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T05:49:28,623 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T05:49:28,624 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,32937,1731217767318-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,624 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,624 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.Replication(171): 4999977c7e1b,32937,1731217767318 started 2024-11-10T05:49:28,643 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:28,644 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(1482): Serving as 4999977c7e1b,32937,1731217767318, RpcServer on 4999977c7e1b/172.17.0.2:32937, sessionid=0x10190ba2cb30001 2024-11-10T05:49:28,644 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T05:49:28,644 DEBUG [RS:0;4999977c7e1b:32937 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4999977c7e1b,32937,1731217767318 2024-11-10T05:49:28,644 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,32937,1731217767318' 2024-11-10T05:49:28,644 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T05:49:28,645 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T05:49:28,646 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T05:49:28,646 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T05:49:28,646 DEBUG [RS:0;4999977c7e1b:32937 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4999977c7e1b,32937,1731217767318 2024-11-10T05:49:28,646 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,32937,1731217767318' 2024-11-10T05:49:28,646 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T05:49:28,647 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T05:49:28,650 DEBUG [RS:0;4999977c7e1b:32937 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T05:49:28,650 INFO [RS:0;4999977c7e1b:32937 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T05:49:28,650 INFO [RS:0;4999977c7e1b:32937 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T05:49:28,699 WARN [4999977c7e1b:43297 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T05:49:28,703 INFO [RS:1;4999977c7e1b:35809 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:49:28,704 INFO [RS:1;4999977c7e1b:35809 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4999977c7e1b%2C35809%2C1731217767455, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,35809,1731217767455, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs, maxLogs=55 2024-11-10T05:49:28,727 INFO [RS:1;4999977c7e1b:35809 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,35809,1731217767455/4999977c7e1b%2C35809%2C1731217767455.1731217768707 2024-11-10T05:49:28,728 DEBUG [RS:1;4999977c7e1b:35809 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:28,751 INFO [RS:0;4999977c7e1b:32937 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:49:28,752 INFO [RS:0;4999977c7e1b:32937 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4999977c7e1b%2C32937%2C1731217767318, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,32937,1731217767318, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs, maxLogs=55 2024-11-10T05:49:28,774 INFO [RS:0;4999977c7e1b:32937 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,32937,1731217767318/4999977c7e1b%2C32937%2C1731217767318.1731217768754 2024-11-10T05:49:28,774 DEBUG [RS:0;4999977c7e1b:32937 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:28,952 DEBUG [4999977c7e1b:43297 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-10T05:49:28,964 DEBUG [4999977c7e1b:43297 {}] balancer.BalancerClusterState(204): Hosts are {4999977c7e1b=0} racks are {/default-rack=0} 2024-11-10T05:49:28,967 DEBUG [4999977c7e1b:43297 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T05:49:28,967 DEBUG [4999977c7e1b:43297 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T05:49:28,967 DEBUG [4999977c7e1b:43297 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T05:49:28,967 DEBUG [4999977c7e1b:43297 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T05:49:28,967 INFO [4999977c7e1b:43297 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T05:49:28,968 INFO [4999977c7e1b:43297 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T05:49:28,968 DEBUG [4999977c7e1b:43297 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T05:49:28,975 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4999977c7e1b,35809,1731217767455 2024-11-10T05:49:28,981 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4999977c7e1b,35809,1731217767455, state=OPENING 2024-11-10T05:49:28,986 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T05:49:28,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:28,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:28,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:28,989 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:28,990 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T05:49:28,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4999977c7e1b,35809,1731217767455}] 2024-11-10T05:49:29,165 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T05:49:29,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54351, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T05:49:29,176 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T05:49:29,177 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:29,178 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T05:49:29,179 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4999977c7e1b%2C35809%2C1731217767455.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,35809,1731217767455, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs, maxLogs=55 2024-11-10T05:49:29,204 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,35809,1731217767455/4999977c7e1b%2C35809%2C1731217767455.meta.1731217769181.meta 2024-11-10T05:49:29,205 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:29,206 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:29,208 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T05:49:29,212 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T05:49:29,218 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T05:49:29,222 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T05:49:29,223 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:29,223 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T05:49:29,223 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T05:49:29,225 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T05:49:29,226 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T05:49:29,226 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:29,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:29,227 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T05:49:29,228 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T05:49:29,228 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:29,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:29,228 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T05:49:29,229 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T05:49:29,229 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:29,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:29,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T05:49:29,230 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T05:49:29,230 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:29,230 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:29,231 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T05:49:29,232 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:29,232 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:29,233 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T05:49:29,233 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T05:49:29,233 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:49:29,235 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T05:49:29,236 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64873162, jitterRate=-0.03331455588340759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:49:29,236 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T05:49:29,237 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731217769224Writing region info on filesystem at 1731217769224Initializing all the Stores at 1731217769225 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217769225Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217769225Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217769225Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217769225Cleaning up temporary data from old regions at 1731217769233 (+8 ms)Running coprocessor post-open hooks at 1731217769236 (+3 ms)Region opened successfully at 1731217769237 (+1 ms) 2024-11-10T05:49:29,244 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731217769156 2024-11-10T05:49:29,255 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T05:49:29,255 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T05:49:29,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4999977c7e1b,35809,1731217767455 2024-11-10T05:49:29,257 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4999977c7e1b,35809,1731217767455, state=OPEN 2024-11-10T05:49:29,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T05:49:29,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T05:49:29,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T05:49:29,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:29,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:29,260 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:29,260 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4999977c7e1b,35809,1731217767455 2024-11-10T05:49:29,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T05:49:29,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4999977c7e1b,35809,1731217767455 in 269 msec 2024-11-10T05:49:29,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T05:49:29,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 731 msec 2024-11-10T05:49:29,268 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:29,268 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T05:49:29,294 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T05:49:29,295 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4999977c7e1b,35809,1731217767455, seqNum=-1] 2024-11-10T05:49:29,315 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T05:49:29,318 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36015, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T05:49:29,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0220 sec 2024-11-10T05:49:29,335 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731217769335, completionTime=-1 2024-11-10T05:49:29,339 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-10T05:49:29,339 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T05:49:29,371 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-10T05:49:29,371 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731217829371 2024-11-10T05:49:29,371 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731217889371 2024-11-10T05:49:29,371 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 32 msec 2024-11-10T05:49:29,373 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-10T05:49:29,380 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,43297,1731217766535-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:29,380 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,43297,1731217766535-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:29,380 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,43297,1731217766535-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:29,382 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4999977c7e1b:43297, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:29,382 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:29,383 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:29,389 DEBUG [master/4999977c7e1b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T05:49:29,411 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.861sec 2024-11-10T05:49:29,413 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T05:49:29,414 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T05:49:29,415 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T05:49:29,416 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T05:49:29,416 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T05:49:29,417 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,43297,1731217766535-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T05:49:29,417 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,43297,1731217766535-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T05:49:29,418 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T05:49:29,419 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T05:49:29,419 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,43297,1731217766535-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:29,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@218aa893, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T05:49:29,522 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T05:49:29,522 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T05:49:29,526 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4999977c7e1b,43297,-1 for getting cluster id 2024-11-10T05:49:29,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T05:49:29,538 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4d71b06b-660c-45bd-a67f-0b1b07ca6d29' 2024-11-10T05:49:29,540 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T05:49:29,540 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4d71b06b-660c-45bd-a67f-0b1b07ca6d29" 2024-11-10T05:49:29,542 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1237d7b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T05:49:29,543 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4999977c7e1b,43297,-1] 2024-11-10T05:49:29,545 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T05:49:29,546 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:29,548 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T05:49:29,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fe5747, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T05:49:29,551 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T05:49:29,558 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4999977c7e1b,35809,1731217767455, seqNum=-1] 2024-11-10T05:49:29,558 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T05:49:29,561 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T05:49:29,580 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4999977c7e1b,43297,1731217766535 2024-11-10T05:49:29,584 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T05:49:29,589 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4999977c7e1b,43297,1731217766535 2024-11-10T05:49:29,592 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d5a7858 2024-11-10T05:49:29,593 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T05:49:29,597 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52656, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T05:49:29,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731217769580', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:49:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 2024-11-10T05:49:29,613 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T05:49:29,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731217769580" procId is: 4 2024-11-10T05:49:29,615 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:29,617 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T05:49:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:29,631 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4b2a6cdcb4de0043a9276067be5c5769, NAME => 'testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731217769580', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:29,638 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:29,638 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(1722): Closing 4b2a6cdcb4de0043a9276067be5c5769, disabling compactions & flushes 2024-11-10T05:49:29,638 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:29,638 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:29,638 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. after waiting 0 ms 2024-11-10T05:49:29,638 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:29,638 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:29,639 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731217769580-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4b2a6cdcb4de0043a9276067be5c5769: Waiting for close lock at 1731217769638Disabling compacts and flushes for region at 1731217769638Disabling writes for close at 1731217769638Writing region close event to WAL at 1731217769638Closed at 1731217769638 2024-11-10T05:49:29,640 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T05:49:29,645 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731217769640"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731217769640"}]},"ts":"1731217769640"} 2024-11-10T05:49:29,649 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T05:49:29,650 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T05:49:29,653 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731217769580","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731217769651"}]},"ts":"1731217769651"} 2024-11-10T05:49:29,657 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731217769580, state=ENABLING in hbase:meta 2024-11-10T05:49:29,658 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4999977c7e1b=0} racks are {/default-rack=0} 2024-11-10T05:49:29,658 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T05:49:29,658 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T05:49:29,658 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T05:49:29,658 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T05:49:29,658 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T05:49:29,658 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T05:49:29,658 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T05:49:29,660 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, ASSIGN}] 2024-11-10T05:49:29,661 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, ASSIGN 2024-11-10T05:49:29,662 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, ASSIGN; state=OFFLINE, location=4999977c7e1b,32937,1731217767318; forceNewPlan=false, retain=false 2024-11-10T05:49:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:29,815 INFO [4999977c7e1b:43297 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T05:49:29,816 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4b2a6cdcb4de0043a9276067be5c5769, regionState=OPENING, regionLocation=4999977c7e1b,32937,1731217767318 2024-11-10T05:49:29,824 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, ASSIGN because future has completed 2024-11-10T05:49:29,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,32937,1731217767318}] 2024-11-10T05:49:29,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:29,979 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T05:49:29,981 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57821, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T05:49:29,984 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:29,984 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b2a6cdcb4de0043a9276067be5c5769, NAME => 'testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:29,984 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731217769580 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,985 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:29,985 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,985 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,986 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,990 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2a6cdcb4de0043a9276067be5c5769 columnFamilyName family 2024-11-10T05:49:29,990 DEBUG [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:29,991 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.HStore(327): Store=4b2a6cdcb4de0043a9276067be5c5769/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:29,991 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,993 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,993 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,993 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,993 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:29,994 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:30,001 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:49:30,002 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 4b2a6cdcb4de0043a9276067be5c5769; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63615737, jitterRate=-0.05205164849758148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T05:49:30,002 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:30,003 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 4b2a6cdcb4de0043a9276067be5c5769: Running coprocessor pre-open hook at 1731217769985Writing region info on filesystem at 1731217769985Initializing all the Stores at 1731217769985Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217769985Cleaning up temporary data from old regions at 1731217769993 (+8 ms)Running coprocessor post-open hooks at 1731217770002 (+9 ms)Region opened successfully at 1731217770003 (+1 ms) 2024-11-10T05:49:30,004 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769., pid=6, masterSystemTime=1731217769978 2024-11-10T05:49:30,007 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:30,007 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:30,008 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=4b2a6cdcb4de0043a9276067be5c5769, regionState=OPEN, openSeqNum=2, regionLocation=4999977c7e1b,32937,1731217767318 2024-11-10T05:49:30,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,32937,1731217767318 because future has completed 2024-11-10T05:49:30,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T05:49:30,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,32937,1731217767318 in 187 msec 2024-11-10T05:49:30,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T05:49:30,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, ASSIGN in 355 msec 2024-11-10T05:49:30,018 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T05:49:30,018 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731217769580","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731217770018"}]},"ts":"1731217770018"} 2024-11-10T05:49:30,021 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731217769580, state=ENABLED in hbase:meta 2024-11-10T05:49:30,022 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T05:49:30,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731217769580 in 416 msec 2024-11-10T05:49:30,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:30,246 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731217769580 get assigned. Timeout = 60000ms 2024-11-10T05:49:30,247 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731217769580 completed 2024-11-10T05:49:30,249 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T05:49:30,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731217769580 assigned to meta. Checking AM states. 2024-11-10T05:49:30,256 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T05:49:30,257 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731217769580 assigned. 2024-11-10T05:49:30,265 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731217769580', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769., hostname=4999977c7e1b,32937,1731217767318, seqNum=2] 2024-11-10T05:49:30,266 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T05:49:30,269 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T05:49:30,280 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4b2a6cdcb4de0043a9276067be5c5769 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T05:49:30,306 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:30,343 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/.tmp/family/c08c240db46e44ab910e18949e9c1224 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T05:49:30,349 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/.tmp/family/c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:30,384 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/.tmp/family/c08c240db46e44ab910e18949e9c1224 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:30,386 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224, delay=954 ms 2024-11-10T05:49:30,387 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T05:49:30,395 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 4b2a6cdcb4de0043a9276067be5c5769 in 112ms, sequenceid=6, compaction requested=false 2024-11-10T05:49:30,396 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731217769580' 2024-11-10T05:49:30,399 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4b2a6cdcb4de0043a9276067be5c5769: 2024-11-10T05:49:31,342 DEBUG [hfile-prefetch-1731217770386 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c08c240db46e44ab910e18949e9c1224_0. 2024-11-10T05:49:31,346 DEBUG [hfile-prefetch-1731217770386 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key c08c240db46e44ab910e18949e9c1224_117. 2024-11-10T05:49:31,347 DEBUG [hfile-prefetch-1731217770386 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224, with total blocks 2, and data blocks 1 2024-11-10T05:49:31,347 DEBUG [hfile-prefetch-1731217770386 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T05:49:31,348 DEBUG [hfile-prefetch-1731217770386 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T05:49:31,348 DEBUG [hfile-prefetch-1731217770386 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224. Will sleep 100ms and try the verification again. 2024-11-10T05:49:31,376 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T05:49:31,376 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T05:49:31,377 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:31,377 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:31,392 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T05:49:31,392 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731217771376 2024-11-10T05:49:31,393 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T05:49:31,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=4b2a6cdcb4de0043a9276067be5c5769, source=4999977c7e1b,32937,1731217767318, destination=4999977c7e1b,35809,1731217767455, warming up region on 4999977c7e1b,35809,1731217767455 2024-11-10T05:49:31,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=4b2a6cdcb4de0043a9276067be5c5769, source=4999977c7e1b,32937,1731217767318, destination=4999977c7e1b,35809,1731217767455, running balancer 2024-11-10T05:49:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, REOPEN/MOVE 2024-11-10T05:49:31,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,413 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, REOPEN/MOVE 2024-11-10T05:49:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(7855): Warmup {ENCODED => 4b2a6cdcb4de0043a9276067be5c5769, NAME => 'testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:31,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:31,414 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,415 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b2a6cdcb4de0043a9276067be5c5769, regionState=CLOSING, regionLocation=4999977c7e1b,32937,1731217767318 2024-11-10T05:49:31,415 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2a6cdcb4de0043a9276067be5c5769 columnFamilyName family 2024-11-10T05:49:31,415 DEBUG [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:31,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, REOPEN/MOVE because future has completed 2024-11-10T05:49:31,418 DEBUG [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:31,418 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.HStore(327): Store=4b2a6cdcb4de0043a9276067be5c5769/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(1722): Closing 4b2a6cdcb4de0043a9276067be5c5769, disabling compactions & flushes 2024-11-10T05:49:31,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,419 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-10T05:49:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. after waiting 0 ms 2024-11-10T05:49:31,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,419 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,32937,1731217767318}] 2024-11-10T05:49:31,420 DEBUG [StoreFileCloser-4b2a6cdcb4de0043a9276067be5c5769-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:31,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35809 {}] regionserver.HRegion(1676): Region close journal for 4b2a6cdcb4de0043a9276067be5c5769: Waiting for close lock at 1731217771419Disabling compacts and flushes for region at 1731217771419Disabling writes for close at 1731217771419Writing region close event to WAL at 1731217771420 (+1 ms)Closed at 1731217771420 2024-11-10T05:49:31,423 WARN [hfile-prefetch-1731217770386 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-10T05:49:31,424 DEBUG [hfile-prefetch-1731217770386 {}] hfile.PrefetchExecutor(121): Prefetch completed for c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43297 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-10T05:49:31,577 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,578 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-10T05:49:31,578 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 4b2a6cdcb4de0043a9276067be5c5769, disabling compactions & flushes 2024-11-10T05:49:31,578 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,578 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,578 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. after waiting 0 ms 2024-11-10T05:49:31,578 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,580 DEBUG [StoreFileCloser-4b2a6cdcb4de0043a9276067be5c5769-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file c08c240db46e44ab910e18949e9c1224, starting offset: 0, end offset: 9223372036854775807 2024-11-10T05:49:31,581 DEBUG [StoreFileCloser-4b2a6cdcb4de0043a9276067be5c5769-family-1 {}] bucket.BucketCache(857): removed key c08c240db46e44ab910e18949e9c1224_0 from back map with offset lock 2094080 in the evict process 2024-11-10T05:49:31,587 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T05:49:31,589 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,589 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 4b2a6cdcb4de0043a9276067be5c5769: Waiting for close lock at 1731217771578Running coprocessor pre-close hooks at 1731217771578Disabling compacts and flushes for region at 1731217771578Disabling writes for close at 1731217771578Writing region close event to WAL at 1731217771582 (+4 ms)Running coprocessor post-close hooks at 1731217771588 (+6 ms)Closed at 1731217771589 (+1 ms) 2024-11-10T05:49:31,589 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 4b2a6cdcb4de0043a9276067be5c5769 move to 4999977c7e1b,35809,1731217767455 record at close sequenceid=6 2024-11-10T05:49:31,592 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,593 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T05:49:31,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b2a6cdcb4de0043a9276067be5c5769, regionState=CLOSED 2024-11-10T05:49:31,593 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T05:49:31,593 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:31,593 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:31,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,32937,1731217767318 because future has completed 2024-11-10T05:49:31,596 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T05:49:31,596 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731217771593 2024-11-10T05:49:31,596 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T05:49:31,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T05:49:31,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,32937,1731217767318 in 179 msec 2024-11-10T05:49:31,600 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, REOPEN/MOVE; state=CLOSED, location=4999977c7e1b,35809,1731217767455; forceNewPlan=false, retain=false 2024-11-10T05:49:31,750 INFO [4999977c7e1b:43297 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T05:49:31,751 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b2a6cdcb4de0043a9276067be5c5769, regionState=OPENING, regionLocation=4999977c7e1b,35809,1731217767455 2024-11-10T05:49:31,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, REOPEN/MOVE because future has completed 2024-11-10T05:49:31,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,35809,1731217767455}] 2024-11-10T05:49:31,911 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,911 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b2a6cdcb4de0043a9276067be5c5769, NAME => 'testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:31,912 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731217769580 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,912 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:31,912 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,912 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,913 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,914 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b2a6cdcb4de0043a9276067be5c5769 columnFamilyName family 2024-11-10T05:49:31,914 DEBUG [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:31,916 DEBUG [StoreFileOpener-4b2a6cdcb4de0043a9276067be5c5769-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224, delay=923 ms 2024-11-10T05:49:31,918 DEBUG [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:31,918 INFO [StoreOpener-4b2a6cdcb4de0043a9276067be5c5769-1 {}] regionserver.HStore(327): Store=4b2a6cdcb4de0043a9276067be5c5769/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:31,918 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,919 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,919 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,919 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,919 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,924 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,924 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 4b2a6cdcb4de0043a9276067be5c5769; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69236419, jitterRate=0.03170304000377655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T05:49:31,924 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,925 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 4b2a6cdcb4de0043a9276067be5c5769: Running coprocessor pre-open hook at 1731217771912Writing region info on filesystem at 1731217771912Initializing all the Stores at 1731217771913 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217771913Cleaning up temporary data from old regions at 1731217771919 (+6 ms)Running coprocessor post-open hooks at 1731217771924 (+5 ms)Region opened successfully at 1731217771925 (+1 ms) 2024-11-10T05:49:31,926 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769., pid=9, masterSystemTime=1731217771908 2024-11-10T05:49:31,929 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,929 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=4b2a6cdcb4de0043a9276067be5c5769, regionState=OPEN, openSeqNum=10, regionLocation=4999977c7e1b,35809,1731217767455 2024-11-10T05:49:31,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,35809,1731217767455 because future has completed 2024-11-10T05:49:31,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-10T05:49:31,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b2a6cdcb4de0043a9276067be5c5769, server=4999977c7e1b,35809,1731217767455 in 178 msec 2024-11-10T05:49:31,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731217769580, region=4b2a6cdcb4de0043a9276067be5c5769, REOPEN/MOVE in 524 msec 2024-11-10T05:49:31,965 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T05:49:31,966 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T05:49:31,966 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:31,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:31,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:31,971 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T05:49:31,971 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T05:49:31,971 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-11-10T05:49:31,972 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4999977c7e1b,43297,1731217766535 2024-11-10T05:49:31,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:31,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:31,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:31,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:31,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:31,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:31,975 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:49:31,975 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:31,975 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T05:49:31,975 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:31,975 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:31,975 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:31,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:31,976 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4999977c7e1b,32937,1731217767318' ***** 2024-11-10T05:49:31,976 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T05:49:31,976 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4999977c7e1b,35809,1731217767455' ***** 2024-11-10T05:49:31,976 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T05:49:31,977 INFO [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T05:49:31,977 INFO [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T05:49:31,977 INFO [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.persistence 2024-11-10T05:49:31,977 INFO [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/bucket.persistence 2024-11-10T05:49:31,977 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T05:49:31,977 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T05:49:31,977 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T05:49:31,978 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T05:49:31,978 INFO [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T05:49:31,978 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T05:49:31,978 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T05:49:31,978 INFO [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T05:49:31,978 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T05:49:31,978 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T05:49:31,988 DEBUG [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1488): Thread RS:1;4999977c7e1b:35809 started persisting bucket cache to file 2024-11-10T05:49:31,988 DEBUG [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:31,988 DEBUG [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1488): Thread RS:0;4999977c7e1b:32937 started persisting bucket cache to file 2024-11-10T05:49:31,988 DEBUG [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:31,988 DEBUG [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:31,988 DEBUG [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:31,992 DEBUG [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T05:49:31,992 DEBUG [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T05:49:31,992 DEBUG [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731217771988 2024-11-10T05:49:31,992 DEBUG [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731217771988 2024-11-10T05:49:31,992 DEBUG [RS:1;4999977c7e1b:35809 {}] bucket.BucketCache(1510): Thread RS:1;4999977c7e1b:35809 finished persisting bucket cache to file, renaming 2024-11-10T05:49:31,992 DEBUG [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1510): Thread RS:0;4999977c7e1b:32937 finished persisting bucket cache to file, renaming 2024-11-10T05:49:31,992 WARN [RS:0;4999977c7e1b:32937 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-10T05:49:31,992 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T05:49:31,992 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T05:49:31,993 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T05:49:31,993 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T05:49:31,993 INFO [RS:1;4999977c7e1b:35809 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T05:49:31,993 INFO [RS:0;4999977c7e1b:32937 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T05:49:31,993 INFO [RS:1;4999977c7e1b:35809 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T05:49:31,993 INFO [RS:0;4999977c7e1b:32937 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T05:49:31,993 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(3091): Received CLOSE for 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,993 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(959): stopping server 4999977c7e1b,32937,1731217767318 2024-11-10T05:49:31,993 INFO [RS:0;4999977c7e1b:32937 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T05:49:31,993 INFO [RS:0;4999977c7e1b:32937 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4999977c7e1b:32937. 2024-11-10T05:49:31,994 DEBUG [RS:0;4999977c7e1b:32937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:31,994 DEBUG [RS:0;4999977c7e1b:32937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:31,994 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(959): stopping server 4999977c7e1b,35809,1731217767455 2024-11-10T05:49:31,994 INFO [RS:1;4999977c7e1b:35809 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T05:49:31,994 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(976): stopping server 4999977c7e1b,32937,1731217767318; all regions closed. 2024-11-10T05:49:31,994 INFO [RS:1;4999977c7e1b:35809 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4999977c7e1b:35809. 2024-11-10T05:49:31,994 DEBUG [RS:1;4999977c7e1b:35809 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:31,994 DEBUG [RS:1;4999977c7e1b:35809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:31,994 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T05:49:31,995 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T05:49:31,995 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T05:49:31,995 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T05:49:31,995 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T05:49:31,995 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4b2a6cdcb4de0043a9276067be5c5769, disabling compactions & flushes 2024-11-10T05:49:31,995 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,995 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,995 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. after waiting 0 ms 2024-11-10T05:49:31,995 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:31,995 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T05:49:31,995 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T05:49:31,995 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1325): Online Regions={4b2a6cdcb4de0043a9276067be5c5769=testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T05:49:31,996 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T05:49:31,996 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T05:49:31,996 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T05:49:31,996 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T05:49:31,996 DEBUG [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4b2a6cdcb4de0043a9276067be5c5769 2024-11-10T05:49:31,996 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T05:49:31,996 DEBUG [StoreFileCloser-4b2a6cdcb4de0043a9276067be5c5769-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/family/c08c240db46e44ab910e18949e9c1224 2024-11-10T05:49:31,996 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-10T05:49:32,003 DEBUG [RS:0;4999977c7e1b:32937 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs 2024-11-10T05:49:32,003 INFO [RS:0;4999977c7e1b:32937 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4999977c7e1b%2C32937%2C1731217767318:(num 1731217768754) 2024-11-10T05:49:32,003 DEBUG [RS:0;4999977c7e1b:32937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:32,003 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:32,003 INFO [RS:0;4999977c7e1b:32937 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T05:49:32,003 INFO [RS:0;4999977c7e1b:32937 {}] hbase.ChoreService(370): Chore service for: regionserver/4999977c7e1b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T05:49:32,004 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnRegionMove1731217769580/4b2a6cdcb4de0043a9276067be5c5769/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-10T05:49:32,004 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T05:49:32,004 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T05:49:32,004 INFO [regionserver/4999977c7e1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T05:49:32,004 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T05:49:32,004 INFO [RS:0;4999977c7e1b:32937 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T05:49:32,004 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:32,004 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4b2a6cdcb4de0043a9276067be5c5769: Waiting for close lock at 1731217771994Running coprocessor pre-close hooks at 1731217771995 (+1 ms)Disabling compacts and flushes for region at 1731217771995Disabling writes for close at 1731217771995Writing region close event to WAL at 1731217771996 (+1 ms)Running coprocessor post-close hooks at 1731217772004 (+8 ms)Closed at 1731217772004 2024-11-10T05:49:32,004 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769. 2024-11-10T05:49:32,004 INFO [RS:0;4999977c7e1b:32937 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:32937 2024-11-10T05:49:32,008 INFO [regionserver/4999977c7e1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:32,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4999977c7e1b,32937,1731217767318 2024-11-10T05:49:32,009 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:32,010 INFO [RS:0;4999977c7e1b:32937 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T05:49:32,011 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4999977c7e1b,32937,1731217767318] 2024-11-10T05:49:32,011 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c8908d6ba38840fcb18ed4ca835d2eeb 2024-11-10T05:49:32,013 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4999977c7e1b,32937,1731217767318 already deleted, retry=false 2024-11-10T05:49:32,013 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4999977c7e1b,32937,1731217767318 expired; onlineServers=1 2024-11-10T05:49:32,031 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/info/c8908d6ba38840fcb18ed4ca835d2eeb is 205, key is testBlockEvictionOnRegionMove1731217769580,,1731217769599.4b2a6cdcb4de0043a9276067be5c5769./info:regioninfo/1731217771929/Put/seqid=0 2024-11-10T05:49:32,032 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/info/c8908d6ba38840fcb18ed4ca835d2eeb 2024-11-10T05:49:32,042 INFO [regionserver/4999977c7e1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:32,044 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ef296ab014454714b6cba0be2f76d5fd 2024-11-10T05:49:32,058 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/ns/ef296ab014454714b6cba0be2f76d5fd is 43, key is default/ns:d/1731217769322/Put/seqid=0 2024-11-10T05:49:32,058 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/ns/ef296ab014454714b6cba0be2f76d5fd 2024-11-10T05:49:32,068 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8c6a195753954c00a0a1afdf0dce4082 2024-11-10T05:49:32,082 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/table/8c6a195753954c00a0a1afdf0dce4082 is 78, key is testBlockEvictionOnRegionMove1731217769580/table:state/1731217770018/Put/seqid=0 2024-11-10T05:49:32,083 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/table/8c6a195753954c00a0a1afdf0dce4082 2024-11-10T05:49:32,086 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/info/c8908d6ba38840fcb18ed4ca835d2eeb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/c8908d6ba38840fcb18ed4ca835d2eeb 2024-11-10T05:49:32,087 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/c8908d6ba38840fcb18ed4ca835d2eeb, delay=985 ms 2024-11-10T05:49:32,088 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/c8908d6ba38840fcb18ed4ca835d2eeb, entries=15, sequenceid=15, filesize=7.6 K 2024-11-10T05:49:32,089 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/ns/ef296ab014454714b6cba0be2f76d5fd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/ef296ab014454714b6cba0be2f76d5fd 2024-11-10T05:49:32,090 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/ef296ab014454714b6cba0be2f76d5fd, delay=936 ms 2024-11-10T05:49:32,092 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/ef296ab014454714b6cba0be2f76d5fd, entries=2, sequenceid=15, filesize=5.0 K 2024-11-10T05:49:32,092 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/table/8c6a195753954c00a0a1afdf0dce4082 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/8c6a195753954c00a0a1afdf0dce4082 2024-11-10T05:49:32,094 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/8c6a195753954c00a0a1afdf0dce4082, delay=926 ms 2024-11-10T05:49:32,095 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/8c6a195753954c00a0a1afdf0dce4082, entries=2, sequenceid=15, filesize=5.3 K 2024-11-10T05:49:32,096 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=15, compaction requested=false 2024-11-10T05:49:32,096 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-10T05:49:32,098 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/c8908d6ba38840fcb18ed4ca835d2eeb 2024-11-10T05:49:32,098 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/ef296ab014454714b6cba0be2f76d5fd 2024-11-10T05:49:32,099 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/8c6a195753954c00a0a1afdf0dce4082 2024-11-10T05:49:32,104 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-10T05:49:32,104 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T05:49:32,104 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T05:49:32,105 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731217771995Running coprocessor pre-close hooks at 1731217771995Disabling compacts and flushes for region at 1731217771995Disabling writes for close at 1731217771996 (+1 ms)Obtaining lock to block concurrent updates at 1731217771996Preparing flush snapshotting stores in 1588230740 at 1731217771996Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731217771997 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731217771997Flushing 1588230740/info: creating writer at 1731217771997Flushing 1588230740/info: appending metadata at 1731217772030 (+33 ms)Flushing 1588230740/info: closing flushed file at 1731217772030Flushing 1588230740/ns: creating writer at 1731217772037 (+7 ms)Flushing 1588230740/ns: appending metadata at 1731217772057 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731217772057Flushing 1588230740/table: creating writer at 1731217772061 (+4 ms)Flushing 1588230740/table: appending metadata at 1731217772082 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731217772082Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4bc23c6: reopening flushed file at 1731217772085 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28d550c7: reopening flushed file at 1731217772089 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f216f07: reopening flushed file at 1731217772092 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=15, compaction requested=false at 1731217772096 (+4 ms)Writing region close event to WAL at 1731217772099 (+3 ms)Running coprocessor post-close hooks at 1731217772104 (+5 ms)Closed at 1731217772104 2024-11-10T05:49:32,105 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T05:49:32,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:32,112 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:32937-0x10190ba2cb30001, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:32,113 INFO [RS:0;4999977c7e1b:32937 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T05:49:32,113 INFO [RS:0;4999977c7e1b:32937 {}] regionserver.HRegionServer(1031): Exiting; stopping=4999977c7e1b,32937,1731217767318; zookeeper connection closed. 2024-11-10T05:49:32,113 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27b08d5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27b08d5 2024-11-10T05:49:32,196 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(976): stopping server 4999977c7e1b,35809,1731217767455; all regions closed. 2024-11-10T05:49:32,199 DEBUG [RS:1;4999977c7e1b:35809 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs 2024-11-10T05:49:32,199 INFO [RS:1;4999977c7e1b:35809 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4999977c7e1b%2C35809%2C1731217767455.meta:.meta(num 1731217769181) 2024-11-10T05:49:32,201 DEBUG [RS:1;4999977c7e1b:35809 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs 2024-11-10T05:49:32,201 INFO [RS:1;4999977c7e1b:35809 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4999977c7e1b%2C35809%2C1731217767455:(num 1731217768707) 2024-11-10T05:49:32,201 DEBUG [RS:1;4999977c7e1b:35809 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:32,201 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:32,201 INFO [RS:1;4999977c7e1b:35809 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T05:49:32,201 INFO [RS:1;4999977c7e1b:35809 {}] hbase.ChoreService(370): Chore service for: regionserver/4999977c7e1b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T05:49:32,201 INFO [RS:1;4999977c7e1b:35809 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T05:49:32,202 INFO [regionserver/4999977c7e1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T05:49:32,202 INFO [RS:1;4999977c7e1b:35809 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35809 2024-11-10T05:49:32,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4999977c7e1b,35809,1731217767455 2024-11-10T05:49:32,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:32,204 INFO [RS:1;4999977c7e1b:35809 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T05:49:32,206 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4999977c7e1b,35809,1731217767455] 2024-11-10T05:49:32,207 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4999977c7e1b,35809,1731217767455 already deleted, retry=false 2024-11-10T05:49:32,207 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4999977c7e1b,35809,1731217767455 expired; onlineServers=0 2024-11-10T05:49:32,207 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4999977c7e1b,43297,1731217766535' ***** 2024-11-10T05:49:32,207 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T05:49:32,208 INFO [M:0;4999977c7e1b:43297 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T05:49:32,208 INFO [M:0;4999977c7e1b:43297 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T05:49:32,208 DEBUG [M:0;4999977c7e1b:43297 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T05:49:32,208 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T05:49:32,208 DEBUG [M:0;4999977c7e1b:43297 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T05:49:32,208 DEBUG [master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.small.0-1731217768436 {}] cleaner.HFileCleaner(306): Exit Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.small.0-1731217768436,5,FailOnTimeoutGroup] 2024-11-10T05:49:32,208 DEBUG [master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.large.0-1731217768434 {}] cleaner.HFileCleaner(306): Exit Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.large.0-1731217768434,5,FailOnTimeoutGroup] 2024-11-10T05:49:32,208 INFO [M:0;4999977c7e1b:43297 {}] hbase.ChoreService(370): Chore service for: master/4999977c7e1b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T05:49:32,209 INFO [M:0;4999977c7e1b:43297 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T05:49:32,209 DEBUG [M:0;4999977c7e1b:43297 {}] master.HMaster(1795): Stopping service threads 2024-11-10T05:49:32,209 INFO [M:0;4999977c7e1b:43297 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:49:32,209 INFO [M:0;4999977c7e1b:43297 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:49:32,209 INFO [M:0;4999977c7e1b:43297 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T05:49:32,210 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T05:49:32,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T05:49:32,210 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,210 DEBUG [M:0;4999977c7e1b:43297 {}] zookeeper.ZKUtil(347): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T05:49:32,210 WARN [M:0;4999977c7e1b:43297 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T05:49:32,211 INFO [M:0;4999977c7e1b:43297 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.lastflushedseqids 2024-11-10T05:49:32,223 INFO [M:0;4999977c7e1b:43297 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:49:32,224 INFO [M:0;4999977c7e1b:43297 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T05:49:32,224 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:49:32,224 INFO [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,224 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,224 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:49:32,224 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,225 INFO [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-10T05:49:32,234 DEBUG [M:0;4999977c7e1b:43297 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 10da4a49363743b7807e3d62c94da58a 2024-11-10T05:49:32,249 DEBUG [M:0;4999977c7e1b:43297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10da4a49363743b7807e3d62c94da58a is 82, key is hbase:meta,,1/info:regioninfo/1731217769255/Put/seqid=0 2024-11-10T05:49:32,250 INFO [M:0;4999977c7e1b:43297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10da4a49363743b7807e3d62c94da58a 2024-11-10T05:49:32,260 DEBUG [M:0;4999977c7e1b:43297 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9e750bb618d54940bec31b3c8e3ba2b1 2024-11-10T05:49:32,275 DEBUG [M:0;4999977c7e1b:43297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9e750bb618d54940bec31b3c8e3ba2b1 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731217770024/Put/seqid=0 2024-11-10T05:49:32,276 INFO [M:0;4999977c7e1b:43297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9e750bb618d54940bec31b3c8e3ba2b1 2024-11-10T05:49:32,284 DEBUG [M:0;4999977c7e1b:43297 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2cdd63a161574fa9b196796767aeab1f 2024-11-10T05:49:32,297 DEBUG [M:0;4999977c7e1b:43297 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2cdd63a161574fa9b196796767aeab1f is 69, key is 4999977c7e1b,32937,1731217767318/rs:state/1731217768579/Put/seqid=0 2024-11-10T05:49:32,298 INFO [M:0;4999977c7e1b:43297 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2cdd63a161574fa9b196796767aeab1f 2024-11-10T05:49:32,299 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/10da4a49363743b7807e3d62c94da58a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10da4a49363743b7807e3d62c94da58a 2024-11-10T05:49:32,301 INFO [M:0;4999977c7e1b:43297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/10da4a49363743b7807e3d62c94da58a, entries=8, sequenceid=84, filesize=5.5 K 2024-11-10T05:49:32,302 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/9e750bb618d54940bec31b3c8e3ba2b1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9e750bb618d54940bec31b3c8e3ba2b1 2024-11-10T05:49:32,304 INFO [M:0;4999977c7e1b:43297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/9e750bb618d54940bec31b3c8e3ba2b1, entries=9, sequenceid=84, filesize=6.5 K 2024-11-10T05:49:32,305 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2cdd63a161574fa9b196796767aeab1f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2cdd63a161574fa9b196796767aeab1f 2024-11-10T05:49:32,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:32,306 INFO [RS:1;4999977c7e1b:35809 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T05:49:32,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35809-0x10190ba2cb30002, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:32,306 INFO [RS:1;4999977c7e1b:35809 {}] regionserver.HRegionServer(1031): Exiting; stopping=4999977c7e1b,35809,1731217767455; zookeeper connection closed. 2024-11-10T05:49:32,306 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2e413539 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2e413539 2024-11-10T05:49:32,307 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-10T05:49:32,307 INFO [M:0;4999977c7e1b:43297 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2cdd63a161574fa9b196796767aeab1f, entries=2, sequenceid=84, filesize=5.1 K 2024-11-10T05:49:32,308 INFO [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=84, compaction requested=false 2024-11-10T05:49:32,310 INFO [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,310 DEBUG [M:0;4999977c7e1b:43297 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731217772224Disabling compacts and flushes for region at 1731217772224Disabling writes for close at 1731217772224Obtaining lock to block concurrent updates at 1731217772225 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731217772225Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731217772225Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731217772225Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731217772226 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731217772249 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731217772249Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731217772253 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731217772274 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731217772274Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731217772278 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731217772296 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731217772297 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1731217772299 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1731217772302 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1731217772304 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 84ms, sequenceid=84, compaction requested=false at 1731217772308 (+4 ms)Writing region close event to WAL at 1731217772310 (+2 ms)Closed at 1731217772310 2024-11-10T05:49:32,312 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T05:49:32,312 INFO [M:0;4999977c7e1b:43297 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T05:49:32,312 INFO [M:0;4999977c7e1b:43297 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:43297 2024-11-10T05:49:32,313 INFO [M:0;4999977c7e1b:43297 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T05:49:32,415 INFO [M:0;4999977c7e1b:43297 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T05:49:32,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:32,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43297-0x10190ba2cb30000, quorum=127.0.0.1:57491, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:32,419 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T05:49:32,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T05:49:32,439 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,448 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/4999977c7e1b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/4999977c7e1b:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731217771916 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731217772090 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/4999977c7e1b:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/4999977c7e1b:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: hfile-prefetch-1731217772087 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731217770386 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=452 (was 456), ProcessCount=11 (was 11), AvailableMemoryMB=8331 (was 8683) 2024-11-10T05:49:32,454 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=452, ProcessCount=11, AvailableMemoryMB=8331 2024-11-10T05:49:32,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/hadoop-log-dir so I do NOT create it in target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1 2024-11-10T05:49:32,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/hadoop-tmp-dir so I do NOT create it in target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1 2024-11-10T05:49:32,455 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,457 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/cluster_1ae9fa42-3b82-81a6-e0f2-3022ad2bcab6, deleteOnExit=true 2024-11-10T05:49:32,460 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/cluster_1ae9fa42-3b82-81a6-e0f2-3022ad2bcab6/zookeeper_0, clientPort=54565, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/cluster_1ae9fa42-3b82-81a6-e0f2-3022ad2bcab6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/cluster_1ae9fa42-3b82-81a6-e0f2-3022ad2bcab6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-10T05:49:32,461 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54565 2024-11-10T05:49:32,473 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd with version=8 2024-11-10T05:49:32,473 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/hbase-staging 2024-11-10T05:49:32,476 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4999977c7e1b:0 server-side Connection retries=45 2024-11-10T05:49:32,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,476 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T05:49:32,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T05:49:32,476 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-10T05:49:32,477 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T05:49:32,477 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38603 2024-11-10T05:49:32,478 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38603 connecting to ZooKeeper ensemble=127.0.0.1:54565 2024-11-10T05:49:32,483 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:386030x0, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T05:49:32,484 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38603-0x10190ba45140000 connected 2024-11-10T05:49:32,498 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,498 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,499 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:32,500 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd, hbase.cluster.distributed=false 2024-11-10T05:49:32,501 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T05:49:32,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38603 2024-11-10T05:49:32,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38603 2024-11-10T05:49:32,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38603 2024-11-10T05:49:32,502 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38603 2024-11-10T05:49:32,503 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38603 2024-11-10T05:49:32,520 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4999977c7e1b:0 server-side Connection retries=45 2024-11-10T05:49:32,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,520 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T05:49:32,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T05:49:32,521 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T05:49:32,521 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T05:49:32,522 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38677 2024-11-10T05:49:32,523 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38677 connecting to ZooKeeper ensemble=127.0.0.1:54565 2024-11-10T05:49:32,523 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,523 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,527 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386770x0, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T05:49:32,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38677-0x10190ba45140001 connected 2024-11-10T05:49:32,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:32,527 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T05:49:32,528 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.cache 2024-11-10T05:49:32,528 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T05:49:32,528 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T05:49:32,528 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T05:49:32,531 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T05:49:32,531 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T05:49:32,532 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T05:49:32,532 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence is complete. 2024-11-10T05:49:32,532 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T05:49:32,534 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T05:49:32,535 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T05:49:32,536 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T05:49:32,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38677 2024-11-10T05:49:32,538 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38677 2024-11-10T05:49:32,543 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38677 2024-11-10T05:49:32,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38677 2024-11-10T05:49:32,544 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38677 2024-11-10T05:49:32,561 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4999977c7e1b:0 server-side Connection retries=45 2024-11-10T05:49:32,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,561 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T05:49:32,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:32,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T05:49:32,561 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T05:49:32,561 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T05:49:32,562 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41229 2024-11-10T05:49:32,563 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41229 connecting to ZooKeeper ensemble=127.0.0.1:54565 2024-11-10T05:49:32,564 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,564 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:412290x0, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T05:49:32,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41229-0x10190ba45140002 connected 2024-11-10T05:49:32,567 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:32,568 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T05:49:32,569 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.cache 2024-11-10T05:49:32,569 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T05:49:32,569 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T05:49:32,569 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T05:49:32,570 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T05:49:32,570 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-10T05:49:32,570 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T05:49:32,570 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T05:49:32,570 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence is complete. 2024-11-10T05:49:32,570 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T05:49:32,571 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-10T05:49:32,573 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T05:49:32,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41229 2024-11-10T05:49:32,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41229 2024-11-10T05:49:32,576 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41229 2024-11-10T05:49:32,577 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41229 2024-11-10T05:49:32,578 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41229 2024-11-10T05:49:32,592 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4999977c7e1b:38603 2024-11-10T05:49:32,592 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4999977c7e1b,38603,1731217772475 2024-11-10T05:49:32,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:32,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:32,594 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:32,595 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4999977c7e1b,38603,1731217772475 2024-11-10T05:49:32,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T05:49:32,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-10T05:49:32,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,598 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T05:49:32,599 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4999977c7e1b,38603,1731217772475 from backup master directory 2024-11-10T05:49:32,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4999977c7e1b,38603,1731217772475 2024-11-10T05:49:32,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:32,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:32,600 WARN [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T05:49:32,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-10T05:49:32,600 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4999977c7e1b,38603,1731217772475 2024-11-10T05:49:32,601 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/hbase.id] with ID: ddf6ad08-099c-40d7-9935-0cc844ba1573 2024-11-10T05:49:32,601 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.tmp/hbase.id 2024-11-10T05:49:32,608 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/hbase.id] 2024-11-10T05:49:32,644 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:32,644 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-10T05:49:32,645 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-10T05:49:32,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,653 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:49:32,654 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:49:32,654 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:32,674 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store 2024-11-10T05:49:32,680 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:32,680 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:49:32,681 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,681 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,681 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:49:32,681 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,681 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:32,681 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731217772680Disabling compacts and flushes for region at 1731217772680Disabling writes for close at 1731217772681 (+1 ms)Writing region close event to WAL at 1731217772681Closed at 1731217772681 2024-11-10T05:49:32,681 WARN [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/.initializing 2024-11-10T05:49:32,681 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/WALs/4999977c7e1b,38603,1731217772475 2024-11-10T05:49:32,687 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:49:32,690 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4999977c7e1b%2C38603%2C1731217772475, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/WALs/4999977c7e1b,38603,1731217772475, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/oldWALs, maxLogs=10 2024-11-10T05:49:32,708 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/WALs/4999977c7e1b,38603,1731217772475/4999977c7e1b%2C38603%2C1731217772475.1731217772690 2024-11-10T05:49:32,709 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:32,709 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:32,709 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:32,709 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,709 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,710 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:49:32,715 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:32,715 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:49:32,719 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:32,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:49:32,723 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:32,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:49:32,727 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,727 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:32,727 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,728 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,728 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,729 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,729 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,729 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:49:32,730 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:49:32,736 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:49:32,736 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59192699, jitterRate=-0.11796005070209503}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:49:32,738 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731217772710Initializing all the Stores at 1731217772710Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217772710Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217772710Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217772710Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217772710Cleaning up temporary data from old regions at 1731217772729 (+19 ms)Region opened successfully at 1731217772737 (+8 ms) 2024-11-10T05:49:32,738 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:49:32,743 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d904086, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4999977c7e1b/172.17.0.2:0 2024-11-10T05:49:32,744 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-10T05:49:32,744 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-10T05:49:32,744 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:49:32,745 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-10T05:49:32,745 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-10T05:49:32,746 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-10T05:49:32,746 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:49:32,748 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-10T05:49:32,749 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-10T05:49:32,751 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-10T05:49:32,751 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-10T05:49:32,752 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-10T05:49:32,753 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-10T05:49:32,754 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-10T05:49:32,755 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-10T05:49:32,756 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-10T05:49:32,758 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-10T05:49:32,759 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-10T05:49:32,762 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-10T05:49:32,763 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-10T05:49:32,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:32,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:32,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:32,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,765 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4999977c7e1b,38603,1731217772475, sessionid=0x10190ba45140000, setting cluster-up flag (Was=false) 2024-11-10T05:49:32,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,773 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-10T05:49:32,775 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4999977c7e1b,38603,1731217772475 2024-11-10T05:49:32,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:32,783 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-10T05:49:32,784 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4999977c7e1b,38603,1731217772475 2024-11-10T05:49:32,785 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-10T05:49:32,786 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:32,786 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-10T05:49:32,787 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-10T05:49:32,787 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4999977c7e1b,38603,1731217772475 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-10T05:49:32,789 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:32,789 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:32,789 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:32,789 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4999977c7e1b:0, corePoolSize=5, maxPoolSize=5 2024-11-10T05:49:32,789 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4999977c7e1b:0, corePoolSize=10, maxPoolSize=10 2024-11-10T05:49:32,790 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,790 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4999977c7e1b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T05:49:32,790 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,796 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:32,796 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-10T05:49:32,796 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,797 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-10T05:49:32,800 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731217802800 2024-11-10T05:49:32,800 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-10T05:49:32,800 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-10T05:49:32,800 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-10T05:49:32,800 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-10T05:49:32,800 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-10T05:49:32,800 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-10T05:49:32,801 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,801 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-10T05:49:32,801 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-10T05:49:32,801 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-10T05:49:32,801 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-10T05:49:32,802 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-10T05:49:32,802 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.large.0-1731217772802,5,FailOnTimeoutGroup] 2024-11-10T05:49:32,802 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.small.0-1731217772802,5,FailOnTimeoutGroup] 2024-11-10T05:49:32,802 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,802 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-10T05:49:32,802 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,802 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,815 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-10T05:49:32,816 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:32,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:32,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T05:49:32,827 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T05:49:32,827 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:32,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T05:49:32,831 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T05:49:32,831 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:32,831 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T05:49:32,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T05:49:32,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:32,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T05:49:32,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T05:49:32,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:32,838 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:32,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T05:49:32,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:32,839 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:32,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T05:49:32,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T05:49:32,840 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:49:32,840 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T05:49:32,846 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:49:32,846 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69593500, jitterRate=0.037023961544036865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:49:32,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731217772822Initializing all the Stores at 1731217772822Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217772822Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217772823 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217772823Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217772823Cleaning up temporary data from old regions at 1731217772840 (+17 ms)Region opened successfully at 1731217772847 (+7 ms) 2024-11-10T05:49:32,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T05:49:32,847 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T05:49:32,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T05:49:32,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T05:49:32,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T05:49:32,848 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T05:49:32,848 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731217772847Disabling compacts and flushes for region at 1731217772847Disabling writes for close at 1731217772847Writing region close event to WAL at 1731217772848 (+1 ms)Closed at 1731217772848 2024-11-10T05:49:32,849 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:32,849 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-10T05:49:32,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-10T05:49:32,850 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T05:49:32,851 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-10T05:49:32,882 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(746): ClusterId : ddf6ad08-099c-40d7-9935-0cc844ba1573 2024-11-10T05:49:32,882 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(746): ClusterId : ddf6ad08-099c-40d7-9935-0cc844ba1573 2024-11-10T05:49:32,882 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T05:49:32,882 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T05:49:32,884 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T05:49:32,884 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T05:49:32,884 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T05:49:32,884 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T05:49:32,887 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T05:49:32,887 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T05:49:32,887 DEBUG [RS:1;4999977c7e1b:41229 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@babde79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4999977c7e1b/172.17.0.2:0 2024-11-10T05:49:32,887 DEBUG [RS:0;4999977c7e1b:38677 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38342a21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4999977c7e1b/172.17.0.2:0 2024-11-10T05:49:32,899 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;4999977c7e1b:41229 2024-11-10T05:49:32,899 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4999977c7e1b:38677 2024-11-10T05:49:32,900 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T05:49:32,900 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T05:49:32,900 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T05:49:32,900 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T05:49:32,900 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T05:49:32,900 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T05:49:32,901 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(2659): reportForDuty to master=4999977c7e1b,38603,1731217772475 with port=41229, startcode=1731217772560 2024-11-10T05:49:32,901 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(2659): reportForDuty to master=4999977c7e1b,38603,1731217772475 with port=38677, startcode=1731217772520 2024-11-10T05:49:32,901 DEBUG [RS:1;4999977c7e1b:41229 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T05:49:32,901 DEBUG [RS:0;4999977c7e1b:38677 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T05:49:32,903 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53413, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T05:49:32,903 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34667, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T05:49:32,904 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38603 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4999977c7e1b,38677,1731217772520 2024-11-10T05:49:32,904 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38603 {}] master.ServerManager(517): Registering regionserver=4999977c7e1b,38677,1731217772520 2024-11-10T05:49:32,906 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38603 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4999977c7e1b,41229,1731217772560 2024-11-10T05:49:32,906 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38603 {}] master.ServerManager(517): Registering regionserver=4999977c7e1b,41229,1731217772560 2024-11-10T05:49:32,906 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:32,906 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T05:49:32,906 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T05:49:32,907 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:32,907 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T05:49:32,907 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T05:49:32,908 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:32,911 DEBUG [RS:0;4999977c7e1b:38677 {}] zookeeper.ZKUtil(111): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4999977c7e1b,38677,1731217772520 2024-11-10T05:49:32,911 WARN [RS:0;4999977c7e1b:38677 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T05:49:32,911 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4999977c7e1b,38677,1731217772520] 2024-11-10T05:49:32,911 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4999977c7e1b,41229,1731217772560] 2024-11-10T05:49:32,911 INFO [RS:0;4999977c7e1b:38677 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:32,911 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,38677,1731217772520 2024-11-10T05:49:32,911 DEBUG [RS:1;4999977c7e1b:41229 {}] zookeeper.ZKUtil(111): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4999977c7e1b,41229,1731217772560 2024-11-10T05:49:32,911 WARN [RS:1;4999977c7e1b:41229 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T05:49:32,911 INFO [RS:1;4999977c7e1b:41229 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:32,912 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,41229,1731217772560 2024-11-10T05:49:32,919 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T05:49:32,921 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T05:49:32,924 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T05:49:32,924 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T05:49:32,925 INFO [RS:0;4999977c7e1b:38677 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T05:49:32,925 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,927 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T05:49:32,927 INFO [RS:1;4999977c7e1b:41229 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T05:49:32,928 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,928 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T05:49:32,928 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T05:49:32,928 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,928 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,928 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,928 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4999977c7e1b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:32,929 DEBUG [RS:0;4999977c7e1b:38677 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:32,929 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4999977c7e1b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,930 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,930 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,930 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,930 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,930 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:32,930 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38677,1731217772520-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:32,930 DEBUG [RS:1;4999977c7e1b:41229 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:32,932 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,932 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,932 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,932 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,932 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,932 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,41229,1731217772560-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T05:49:32,946 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T05:49:32,946 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38677,1731217772520-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,946 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,946 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.Replication(171): 4999977c7e1b,38677,1731217772520 started 2024-11-10T05:49:32,947 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T05:49:32,947 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,41229,1731217772560-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,947 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,947 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.Replication(171): 4999977c7e1b,41229,1731217772560 started 2024-11-10T05:49:32,961 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,961 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1482): Serving as 4999977c7e1b,38677,1731217772520, RpcServer on 4999977c7e1b/172.17.0.2:38677, sessionid=0x10190ba45140001 2024-11-10T05:49:32,962 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T05:49:32,962 DEBUG [RS:0;4999977c7e1b:38677 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4999977c7e1b,38677,1731217772520 2024-11-10T05:49:32,962 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,38677,1731217772520' 2024-11-10T05:49:32,962 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T05:49:32,963 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T05:49:32,963 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T05:49:32,963 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T05:49:32,963 DEBUG [RS:0;4999977c7e1b:38677 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4999977c7e1b,38677,1731217772520 2024-11-10T05:49:32,963 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,38677,1731217772520' 2024-11-10T05:49:32,963 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T05:49:32,964 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T05:49:32,964 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:32,964 DEBUG [RS:0;4999977c7e1b:38677 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T05:49:32,964 INFO [RS:0;4999977c7e1b:38677 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T05:49:32,964 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1482): Serving as 4999977c7e1b,41229,1731217772560, RpcServer on 4999977c7e1b/172.17.0.2:41229, sessionid=0x10190ba45140002 2024-11-10T05:49:32,964 INFO [RS:0;4999977c7e1b:38677 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T05:49:32,964 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T05:49:32,965 DEBUG [RS:1;4999977c7e1b:41229 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4999977c7e1b,41229,1731217772560 2024-11-10T05:49:32,965 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,41229,1731217772560' 2024-11-10T05:49:32,965 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T05:49:32,965 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T05:49:32,965 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T05:49:32,965 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T05:49:32,966 DEBUG [RS:1;4999977c7e1b:41229 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4999977c7e1b,41229,1731217772560 2024-11-10T05:49:32,966 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,41229,1731217772560' 2024-11-10T05:49:32,966 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T05:49:32,966 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T05:49:32,967 DEBUG [RS:1;4999977c7e1b:41229 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T05:49:32,967 INFO [RS:1;4999977c7e1b:41229 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T05:49:32,967 INFO [RS:1;4999977c7e1b:41229 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T05:49:33,001 WARN [4999977c7e1b:38603 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-10T05:49:33,065 INFO [RS:0;4999977c7e1b:38677 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:49:33,066 INFO [RS:0;4999977c7e1b:38677 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4999977c7e1b%2C38677%2C1731217772520, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,38677,1731217772520, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs, maxLogs=55 2024-11-10T05:49:33,067 INFO [RS:1;4999977c7e1b:41229 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:49:33,068 INFO [RS:1;4999977c7e1b:41229 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4999977c7e1b%2C41229%2C1731217772560, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,41229,1731217772560, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs, maxLogs=55 2024-11-10T05:49:33,087 INFO [RS:1;4999977c7e1b:41229 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,41229,1731217772560/4999977c7e1b%2C41229%2C1731217772560.1731217773069 2024-11-10T05:49:33,088 DEBUG [RS:1;4999977c7e1b:41229 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:33,090 INFO [RS:0;4999977c7e1b:38677 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,38677,1731217772520/4999977c7e1b%2C38677%2C1731217772520.1731217773068 2024-11-10T05:49:33,090 DEBUG [RS:0;4999977c7e1b:38677 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:33,252 DEBUG [4999977c7e1b:38603 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-10T05:49:33,252 DEBUG [4999977c7e1b:38603 {}] balancer.BalancerClusterState(204): Hosts are {4999977c7e1b=0} racks are {/default-rack=0} 2024-11-10T05:49:33,253 DEBUG [4999977c7e1b:38603 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T05:49:33,253 DEBUG [4999977c7e1b:38603 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T05:49:33,253 DEBUG [4999977c7e1b:38603 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T05:49:33,253 DEBUG [4999977c7e1b:38603 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T05:49:33,253 INFO [4999977c7e1b:38603 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T05:49:33,253 INFO [4999977c7e1b:38603 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T05:49:33,253 DEBUG [4999977c7e1b:38603 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T05:49:33,254 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4999977c7e1b,38677,1731217772520 2024-11-10T05:49:33,255 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4999977c7e1b,38677,1731217772520, state=OPENING 2024-11-10T05:49:33,256 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-10T05:49:33,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:33,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:33,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:33,259 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-10T05:49:33,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:33,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:33,259 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:33,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4999977c7e1b,38677,1731217772520}] 2024-11-10T05:49:33,412 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T05:49:33,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44729, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T05:49:33,416 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-10T05:49:33,416 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:33,417 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-10T05:49:33,418 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4999977c7e1b%2C38677%2C1731217772520.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,38677,1731217772520, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs, maxLogs=55 2024-11-10T05:49:33,437 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,38677,1731217772520/4999977c7e1b%2C38677%2C1731217772520.meta.1731217773419.meta 2024-11-10T05:49:33,437 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:33,438 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:33,438 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-10T05:49:33,438 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-10T05:49:33,438 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-10T05:49:33,439 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-10T05:49:33,439 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:33,439 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-10T05:49:33,439 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-10T05:49:33,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-10T05:49:33,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-10T05:49:33,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:33,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:33,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-10T05:49:33,442 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-10T05:49:33,442 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:33,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:33,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-10T05:49:33,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-10T05:49:33,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:33,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:33,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-10T05:49:33,444 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-10T05:49:33,444 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:33,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:49:33,444 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-10T05:49:33,445 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:33,445 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740 2024-11-10T05:49:33,446 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-10T05:49:33,446 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-10T05:49:33,446 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:49:33,447 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-10T05:49:33,447 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61990607, jitterRate=-0.07626797258853912}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:49:33,447 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-10T05:49:33,448 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731217773439Writing region info on filesystem at 1731217773439Initializing all the Stores at 1731217773439Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217773439Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217773440 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217773440Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731217773440Cleaning up temporary data from old regions at 1731217773446 (+6 ms)Running coprocessor post-open hooks at 1731217773447 (+1 ms)Region opened successfully at 1731217773448 (+1 ms) 2024-11-10T05:49:33,448 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731217773411 2024-11-10T05:49:33,451 DEBUG [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-10T05:49:33,451 INFO [RS_OPEN_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-10T05:49:33,452 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4999977c7e1b,38677,1731217772520 2024-11-10T05:49:33,452 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4999977c7e1b,38677,1731217772520, state=OPEN 2024-11-10T05:49:33,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T05:49:33,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T05:49:33,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-10T05:49:33,454 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4999977c7e1b,38677,1731217772520 2024-11-10T05:49:33,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:33,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:33,454 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-10T05:49:33,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-10T05:49:33,456 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4999977c7e1b,38677,1731217772520 in 195 msec 2024-11-10T05:49:33,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-10T05:49:33,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 607 msec 2024-11-10T05:49:33,459 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-10T05:49:33,459 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-10T05:49:33,462 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T05:49:33,463 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4999977c7e1b,38677,1731217772520, seqNum=-1] 2024-11-10T05:49:33,463 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T05:49:33,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42743, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T05:49:33,470 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 684 msec 2024-11-10T05:49:33,470 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731217773470, completionTime=-1 2024-11-10T05:49:33,470 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-10T05:49:33,470 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-10T05:49:33,473 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-10T05:49:33,473 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731217833473 2024-11-10T05:49:33,473 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731217893473 2024-11-10T05:49:33,473 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-10T05:49:33,473 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38603,1731217772475-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:33,473 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38603,1731217772475-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:33,473 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38603,1731217772475-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:33,474 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4999977c7e1b:38603, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:33,474 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:33,474 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:33,476 DEBUG [master/4999977c7e1b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.879sec 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38603,1731217772475-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T05:49:33,479 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38603,1731217772475-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-10T05:49:33,480 DEBUG [master/4999977c7e1b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-10T05:49:33,480 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-10T05:49:33,481 INFO [master/4999977c7e1b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,38603,1731217772475-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:33,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T05:49:33,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4999977c7e1b,38603,-1 for getting cluster id 2024-11-10T05:49:33,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-10T05:49:33,484 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ddf6ad08-099c-40d7-9935-0cc844ba1573' 2024-11-10T05:49:33,484 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-10T05:49:33,485 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ddf6ad08-099c-40d7-9935-0cc844ba1573" 2024-11-10T05:49:33,485 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f832ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T05:49:33,485 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4999977c7e1b,38603,-1] 2024-11-10T05:49:33,485 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-10T05:49:33,486 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:33,487 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-10T05:49:33,489 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fb9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-10T05:49:33,489 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-10T05:49:33,490 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4999977c7e1b,38677,1731217772520, seqNum=-1] 2024-11-10T05:49:33,491 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T05:49:33,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T05:49:33,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4999977c7e1b,38603,1731217772475 2024-11-10T05:49:33,496 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-10T05:49:33,497 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 4999977c7e1b,38603,1731217772475 2024-11-10T05:49:33,497 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@260f2f90 2024-11-10T05:49:33,497 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-10T05:49:33,499 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-10T05:49:33,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38603 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731217773496', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:49:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38603 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 2024-11-10T05:49:33,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-10T05:49:33,503 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:33,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38603 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731217773496" procId is: 4 2024-11-10T05:49:33,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-10T05:49:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:33,515 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f6aa855be653d4feae71ddefde929d46, NAME => 'testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731217773496', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:33,522 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:33,522 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(1722): Closing f6aa855be653d4feae71ddefde929d46, disabling compactions & flushes 2024-11-10T05:49:33,522 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:33,522 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:33,522 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. after waiting 0 ms 2024-11-10T05:49:33,522 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:33,522 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:33,522 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731217773496-pool-0 {}] regionserver.HRegion(1676): Region close journal for f6aa855be653d4feae71ddefde929d46: Waiting for close lock at 1731217773522Disabling compacts and flushes for region at 1731217773522Disabling writes for close at 1731217773522Writing region close event to WAL at 1731217773522Closed at 1731217773522 2024-11-10T05:49:33,524 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 execute state=CREATE_TABLE_ADD_TO_META 2024-11-10T05:49:33,524 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731217773524"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731217773524"}]},"ts":"1731217773524"} 2024-11-10T05:49:33,527 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-10T05:49:33,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-10T05:49:33,528 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731217773496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731217773527"}]},"ts":"1731217773527"} 2024-11-10T05:49:33,530 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731217773496, state=ENABLING in hbase:meta 2024-11-10T05:49:33,530 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {4999977c7e1b=0} racks are {/default-rack=0} 2024-11-10T05:49:33,530 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-10T05:49:33,531 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-10T05:49:33,531 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-10T05:49:33,531 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-10T05:49:33,531 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-10T05:49:33,531 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-10T05:49:33,531 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-10T05:49:33,531 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN}] 2024-11-10T05:49:33,532 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN 2024-11-10T05:49:33,533 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN; state=OFFLINE, location=4999977c7e1b,41229,1731217772560; forceNewPlan=false, retain=false 2024-11-10T05:49:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:33,684 INFO [4999977c7e1b:38603 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-10T05:49:33,684 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f6aa855be653d4feae71ddefde929d46, regionState=OPENING, regionLocation=4999977c7e1b,41229,1731217772560 2024-11-10T05:49:33,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN because future has completed 2024-11-10T05:49:33,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6aa855be653d4feae71ddefde929d46, server=4999977c7e1b,41229,1731217772560}] 2024-11-10T05:49:33,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:33,839 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-10T05:49:33,841 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47327, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-10T05:49:33,843 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:33,843 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f6aa855be653d4feae71ddefde929d46, NAME => 'testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:33,844 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731217773496 f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,844 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:33,844 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,844 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,845 INFO [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,848 INFO [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6aa855be653d4feae71ddefde929d46 columnFamilyName family 2024-11-10T05:49:33,849 DEBUG [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:33,849 INFO [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] regionserver.HStore(327): Store=f6aa855be653d4feae71ddefde929d46/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:33,849 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,850 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,850 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,850 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,850 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,851 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,857 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:49:33,857 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f6aa855be653d4feae71ddefde929d46; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66794704, jitterRate=-0.00468134880065918}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T05:49:33,857 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:33,858 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f6aa855be653d4feae71ddefde929d46: Running coprocessor pre-open hook at 1731217773844Writing region info on filesystem at 1731217773844Initializing all the Stores at 1731217773844Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217773844Cleaning up temporary data from old regions at 1731217773850 (+6 ms)Running coprocessor post-open hooks at 1731217773857 (+7 ms)Region opened successfully at 1731217773858 (+1 ms) 2024-11-10T05:49:33,859 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46., pid=6, masterSystemTime=1731217773839 2024-11-10T05:49:33,861 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:33,861 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:33,862 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f6aa855be653d4feae71ddefde929d46, regionState=OPEN, openSeqNum=2, regionLocation=4999977c7e1b,41229,1731217772560 2024-11-10T05:49:33,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6aa855be653d4feae71ddefde929d46, server=4999977c7e1b,41229,1731217772560 because future has completed 2024-11-10T05:49:33,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-10T05:49:33,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f6aa855be653d4feae71ddefde929d46, server=4999977c7e1b,41229,1731217772560 in 177 msec 2024-11-10T05:49:33,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-10T05:49:33,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN in 336 msec 2024-11-10T05:49:33,870 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-10T05:49:33,870 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731217773496","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731217773870"}]},"ts":"1731217773870"} 2024-11-10T05:49:33,872 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731217773496, state=ENABLED in hbase:meta 2024-11-10T05:49:33,873 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 execute state=CREATE_TABLE_POST_OPERATION 2024-11-10T05:49:33,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731217773496 in 372 msec 2024-11-10T05:49:34,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38603 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-10T05:49:34,136 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731217773496 completed 2024-11-10T05:49:34,136 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731217773496 get assigned. Timeout = 60000ms 2024-11-10T05:49:34,136 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T05:49:34,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731217773496 assigned to meta. Checking AM states. 2024-11-10T05:49:34,140 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-10T05:49:34,140 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731217773496 assigned. 2024-11-10T05:49:34,144 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731217773496', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46., hostname=4999977c7e1b,41229,1731217772560, seqNum=2] 2024-11-10T05:49:34,145 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-10T05:49:34,147 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-10T05:49:34,151 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f6aa855be653d4feae71ddefde929d46 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-10T05:49:34,162 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d2fb3c87bf0b4ab18a6c18985157bd88 2024-11-10T05:49:34,177 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/.tmp/family/d2fb3c87bf0b4ab18a6c18985157bd88 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-10T05:49:34,177 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/.tmp/family/d2fb3c87bf0b4ab18a6c18985157bd88 2024-11-10T05:49:34,181 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/.tmp/family/d2fb3c87bf0b4ab18a6c18985157bd88 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88 2024-11-10T05:49:34,183 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88, delay=946 ms 2024-11-10T05:49:34,185 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88, entries=2, sequenceid=6, filesize=4.9 K 2024-11-10T05:49:34,185 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for f6aa855be653d4feae71ddefde929d46 in 34ms, sequenceid=6, compaction requested=false 2024-11-10T05:49:34,186 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731217773496' 2024-11-10T05:49:34,186 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f6aa855be653d4feae71ddefde929d46: 2024-11-10T05:49:34,725 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T05:49:35,130 DEBUG [hfile-prefetch-1731217771916 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d2fb3c87bf0b4ab18a6c18985157bd88_0. 2024-11-10T05:49:35,131 DEBUG [hfile-prefetch-1731217771916 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key d2fb3c87bf0b4ab18a6c18985157bd88_117. 2024-11-10T05:49:35,131 DEBUG [hfile-prefetch-1731217771916 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88, with total blocks 2, and data blocks 1 2024-11-10T05:49:35,131 DEBUG [hfile-prefetch-1731217771916 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-10T05:49:35,132 DEBUG [hfile-prefetch-1731217771916 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-10T05:49:35,132 DEBUG [hfile-prefetch-1731217771916 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88. Will sleep 100ms and try the verification again. 2024-11-10T05:49:35,171 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-10T05:49:35,172 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-10T05:49:35,172 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:35,172 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:35,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-10T05:49:35,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731217775172 2024-11-10T05:49:35,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-10T05:49:35,187 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;4999977c7e1b:41229,5,FailOnTimeoutGroup] 2024-11-10T05:49:35,187 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4999977c7e1b,41229,1731217772560' ***** 2024-11-10T05:49:35,187 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-10T05:49:35,187 INFO [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T05:49:35,187 INFO [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence 2024-11-10T05:49:35,187 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T05:49:35,187 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T05:49:35,187 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T05:49:35,187 INFO [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T05:49:35,188 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T05:49:35,198 DEBUG [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1488): Thread RS:1;4999977c7e1b:41229 started persisting bucket cache to file 2024-11-10T05:49:35,198 DEBUG [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:35,198 DEBUG [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:35,201 DEBUG [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-10T05:49:35,201 DEBUG [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731217775198 2024-11-10T05:49:35,201 DEBUG [RS:1;4999977c7e1b:41229 {}] bucket.BucketCache(1510): Thread RS:1;4999977c7e1b:41229 finished persisting bucket cache to file, renaming 2024-11-10T05:49:35,201 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T05:49:35,201 INFO [RS:1;4999977c7e1b:41229 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T05:49:35,201 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T05:49:35,201 INFO [RS:1;4999977c7e1b:41229 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T05:49:35,201 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(3091): Received CLOSE for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,202 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(959): stopping server 4999977c7e1b,41229,1731217772560 2024-11-10T05:49:35,202 INFO [RS:1;4999977c7e1b:41229 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T05:49:35,202 INFO [RS:1;4999977c7e1b:41229 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;4999977c7e1b:41229. 2024-11-10T05:49:35,202 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T05:49:35,202 DEBUG [RS:1;4999977c7e1b:41229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:35,202 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f6aa855be653d4feae71ddefde929d46, disabling compactions & flushes 2024-11-10T05:49:35,202 DEBUG [RS:1;4999977c7e1b:41229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:35,202 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,202 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,202 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. after waiting 0 ms 2024-11-10T05:49:35,202 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-10T05:49:35,202 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,202 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1325): Online Regions={f6aa855be653d4feae71ddefde929d46=testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.} 2024-11-10T05:49:35,202 DEBUG [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1351): Waiting on f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,203 DEBUG [StoreFileCloser-f6aa855be653d4feae71ddefde929d46-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88 2024-11-10T05:49:35,203 WARN [hfile-prefetch-1731217771916 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-10T05:49:35,203 DEBUG [hfile-prefetch-1731217771916 {}] hfile.PrefetchExecutor(121): Prefetch completed for d2fb3c87bf0b4ab18a6c18985157bd88 2024-11-10T05:49:35,207 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-10T05:49:35,208 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,208 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f6aa855be653d4feae71ddefde929d46: Waiting for close lock at 1731217775202Running coprocessor pre-close hooks at 1731217775202Disabling compacts and flushes for region at 1731217775202Disabling writes for close at 1731217775202Writing region close event to WAL at 1731217775203 (+1 ms)Running coprocessor post-close hooks at 1731217775208 (+5 ms)Closed at 1731217775208 2024-11-10T05:49:35,208 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,234 INFO [regionserver/4999977c7e1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:35,402 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(976): stopping server 4999977c7e1b,41229,1731217772560; all regions closed. 2024-11-10T05:49:35,404 DEBUG [RS:1;4999977c7e1b:41229 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4999977c7e1b%2C41229%2C1731217772560:(num 1731217773069) 2024-11-10T05:49:35,405 DEBUG [RS:1;4999977c7e1b:41229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] hbase.ChoreService(370): Chore service for: regionserver/4999977c7e1b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T05:49:35,405 INFO [regionserver/4999977c7e1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T05:49:35,405 INFO [RS:1;4999977c7e1b:41229 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41229 2024-11-10T05:49:35,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4999977c7e1b,41229,1731217772560 2024-11-10T05:49:35,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:35,409 INFO [RS:1;4999977c7e1b:41229 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T05:49:35,411 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4999977c7e1b,41229,1731217772560] 2024-11-10T05:49:35,412 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4999977c7e1b,41229,1731217772560 already deleted, retry=false 2024-11-10T05:49:35,412 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 4999977c7e1b,41229,1731217772560 on 4999977c7e1b,38603,1731217772475 2024-11-10T05:49:35,417 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 4999977c7e1b,41229,1731217772560, splitWal=true, meta=false 2024-11-10T05:49:35,419 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 4999977c7e1b,41229,1731217772560, splitWal=true, meta=false 2024-11-10T05:49:35,420 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 4999977c7e1b,41229,1731217772560 (carryingMeta=false) 4999977c7e1b,41229,1731217772560/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@10f0b54b[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-10T05:49:35,421 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 4999977c7e1b,41229,1731217772560 had 1 regions 2024-11-10T05:49:35,422 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 4999977c7e1b,41229,1731217772560, splitWal=true, meta=false, isMeta: false 2024-11-10T05:49:35,422 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,41229,1731217772560-splitting 2024-11-10T05:49:35,423 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,41229,1731217772560-splitting dir is empty, no logs to split. 2024-11-10T05:49:35,423 INFO [PEWorker-2 {}] master.SplitWALManager(105): 4999977c7e1b,41229,1731217772560 WAL count=0, meta=false 2024-11-10T05:49:35,424 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,41229,1731217772560-splitting dir is empty, no logs to split. 2024-11-10T05:49:35,425 INFO [PEWorker-2 {}] master.SplitWALManager(105): 4999977c7e1b,41229,1731217772560 WAL count=0, meta=false 2024-11-10T05:49:35,425 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 4999977c7e1b,41229,1731217772560 WAL splitting is done? wals=0, meta=false 2024-11-10T05:49:35,425 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 4999977c7e1b,41229,1731217772560 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,41229,1731217772560-splitting does not exist 2024-11-10T05:49:35,427 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN}] 2024-11-10T05:49:35,428 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN 2024-11-10T05:49:35,428 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-10T05:49:35,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:35,511 INFO [RS:1;4999977c7e1b:41229 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T05:49:35,511 INFO [RS:1;4999977c7e1b:41229 {}] regionserver.HRegionServer(1031): Exiting; stopping=4999977c7e1b,41229,1731217772560; zookeeper connection closed. 2024-11-10T05:49:35,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41229-0x10190ba45140002, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:35,512 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3542d5c5 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3542d5c5 2024-11-10T05:49:35,579 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f6aa855be653d4feae71ddefde929d46, regionState=OPENING, regionLocation=4999977c7e1b,38677,1731217772520 2024-11-10T05:49:35,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN because future has completed 2024-11-10T05:49:35,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6aa855be653d4feae71ddefde929d46, server=4999977c7e1b,38677,1731217772520}] 2024-11-10T05:49:35,703 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4999977c7e1b:0 server-side Connection retries=45 2024-11-10T05:49:35,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:35,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:35,703 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-10T05:49:35,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-10T05:49:35,703 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-10T05:49:35,703 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-10T05:49:35,704 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-10T05:49:35,704 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39971 2024-11-10T05:49:35,706 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39971 connecting to ZooKeeper ensemble=127.0.0.1:54565 2024-11-10T05:49:35,706 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:35,706 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:35,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399710x0, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-10T05:49:35,709 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39971-0x10190ba45140003 connected 2024-11-10T05:49:35,709 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-10T05:49:35,709 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-10T05:49:35,710 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-10T05:49:35,712 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.cache 2024-11-10T05:49:35,712 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-10T05:49:35,712 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-10T05:49:35,712 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-10T05:49:35,713 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-10T05:49:35,713 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-10T05:49:35,713 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-10T05:49:35,713 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-10T05:49:35,714 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-10T05:49:35,715 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-10T05:49:35,715 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-10T05:49:35,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39971 2024-11-10T05:49:35,716 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39971 2024-11-10T05:49:35,720 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39971 2024-11-10T05:49:35,720 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-10T05:49:35,720 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-10T05:49:35,721 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-10T05:49:35,721 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence is complete. 2024-11-10T05:49:35,724 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39971 2024-11-10T05:49:35,725 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39971 2024-11-10T05:49:35,727 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(746): ClusterId : ddf6ad08-099c-40d7-9935-0cc844ba1573 2024-11-10T05:49:35,727 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-10T05:49:35,730 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-10T05:49:35,730 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-10T05:49:35,732 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-10T05:49:35,732 DEBUG [RS:2;4999977c7e1b:39971 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61a1864a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4999977c7e1b/172.17.0.2:0 2024-11-10T05:49:35,737 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,737 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => f6aa855be653d4feae71ddefde929d46, NAME => 'testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:49:35,738 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731217773496 f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,738 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:49:35,738 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,738 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,739 INFO [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,739 INFO [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f6aa855be653d4feae71ddefde929d46 columnFamilyName family 2024-11-10T05:49:35,739 DEBUG [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:49:35,741 DEBUG [StoreFileOpener-f6aa855be653d4feae71ddefde929d46-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88, delay=928 ms 2024-11-10T05:49:35,742 DEBUG [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88 2024-11-10T05:49:35,742 INFO [StoreOpener-f6aa855be653d4feae71ddefde929d46-1 {}] regionserver.HStore(327): Store=f6aa855be653d4feae71ddefde929d46/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:49:35,742 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,743 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,743 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,744 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,744 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,744 DEBUG [RS:2;4999977c7e1b:39971 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;4999977c7e1b:39971 2024-11-10T05:49:35,744 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-10T05:49:35,744 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-10T05:49:35,744 DEBUG [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-10T05:49:35,745 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,745 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(2659): reportForDuty to master=4999977c7e1b,38603,1731217772475 with port=39971, startcode=1731217775703 2024-11-10T05:49:35,745 DEBUG [RS:2;4999977c7e1b:39971 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-10T05:49:35,745 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened f6aa855be653d4feae71ddefde929d46; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74853527, jitterRate=0.11540447175502777}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-10T05:49:35,746 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:35,747 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for f6aa855be653d4feae71ddefde929d46: Running coprocessor pre-open hook at 1731217775738Writing region info on filesystem at 1731217775738Initializing all the Stores at 1731217775738Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731217775738Cleaning up temporary data from old regions at 1731217775744 (+6 ms)Running coprocessor post-open hooks at 1731217775746 (+2 ms)Region opened successfully at 1731217775747 (+1 ms) 2024-11-10T05:49:35,748 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46., pid=9, masterSystemTime=1731217775735 2024-11-10T05:49:35,748 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41969, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-10T05:49:35,748 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38603 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4999977c7e1b,39971,1731217775703 2024-11-10T05:49:35,748 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38603 {}] master.ServerManager(517): Registering regionserver=4999977c7e1b,39971,1731217775703 2024-11-10T05:49:35,750 DEBUG [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd 2024-11-10T05:49:35,750 DEBUG [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-10T05:49:35,750 DEBUG [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-10T05:49:35,751 DEBUG [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,751 INFO [RS_OPEN_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:35,751 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=f6aa855be653d4feae71ddefde929d46, regionState=OPEN, openSeqNum=10, regionLocation=4999977c7e1b,38677,1731217772520 2024-11-10T05:49:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:35,753 DEBUG [RS:2;4999977c7e1b:39971 {}] zookeeper.ZKUtil(111): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4999977c7e1b,39971,1731217775703 2024-11-10T05:49:35,753 WARN [RS:2;4999977c7e1b:39971 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-10T05:49:35,753 INFO [RS:2;4999977c7e1b:39971 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:49:35,753 DEBUG [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,39971,1731217775703 2024-11-10T05:49:35,753 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4999977c7e1b,39971,1731217775703] 2024-11-10T05:49:35,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure f6aa855be653d4feae71ddefde929d46, server=4999977c7e1b,38677,1731217772520 because future has completed 2024-11-10T05:49:35,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-10T05:49:35,757 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure f6aa855be653d4feae71ddefde929d46, server=4999977c7e1b,38677,1731217772520 in 173 msec 2024-11-10T05:49:35,758 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-10T05:49:35,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-10T05:49:35,759 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731217773496, region=f6aa855be653d4feae71ddefde929d46, ASSIGN in 330 msec 2024-11-10T05:49:35,759 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 4999977c7e1b,41229,1731217772560 after splitting done 2024-11-10T05:49:35,760 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-10T05:49:35,761 INFO [RS:2;4999977c7e1b:39971 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-10T05:49:35,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 4999977c7e1b,41229,1731217772560, splitWal=true, meta=false in 346 msec 2024-11-10T05:49:35,761 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,761 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-10T05:49:35,761 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-10T05:49:35,762 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4999977c7e1b:0, corePoolSize=2, maxPoolSize=2 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4999977c7e1b:0, corePoolSize=1, maxPoolSize=1 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:35,762 DEBUG [RS:2;4999977c7e1b:39971 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4999977c7e1b:0, corePoolSize=3, maxPoolSize=3 2024-11-10T05:49:35,763 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,763 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,763 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,763 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,763 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,763 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,39971,1731217775703-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-10T05:49:35,779 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-10T05:49:35,780 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=4999977c7e1b,39971,1731217775703-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,780 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,780 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.Replication(171): 4999977c7e1b,39971,1731217775703 started 2024-11-10T05:49:35,795 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-10T05:49:35,795 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(1482): Serving as 4999977c7e1b,39971,1731217775703, RpcServer on 4999977c7e1b/172.17.0.2:39971, sessionid=0x10190ba45140003 2024-11-10T05:49:35,795 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-10T05:49:35,795 DEBUG [RS:2;4999977c7e1b:39971 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4999977c7e1b,39971,1731217775703 2024-11-10T05:49:35,795 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,39971,1731217775703' 2024-11-10T05:49:35,795 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-10T05:49:35,796 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-10T05:49:35,796 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-10T05:49:35,796 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-10T05:49:35,796 DEBUG [RS:2;4999977c7e1b:39971 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4999977c7e1b,39971,1731217775703 2024-11-10T05:49:35,797 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4999977c7e1b,39971,1731217775703' 2024-11-10T05:49:35,797 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-10T05:49:35,797 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-10T05:49:35,797 DEBUG [RS:2;4999977c7e1b:39971 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-10T05:49:35,797 INFO [RS:2;4999977c7e1b:39971 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-10T05:49:35,797 INFO [RS:2;4999977c7e1b:39971 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-10T05:49:35,898 INFO [RS:2;4999977c7e1b:39971 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:49:35,899 INFO [RS:2;4999977c7e1b:39971 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=4999977c7e1b%2C39971%2C1731217775703, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,39971,1731217775703, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs, maxLogs=55 2024-11-10T05:49:35,917 INFO [RS:2;4999977c7e1b:39971 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/WALs/4999977c7e1b,39971,1731217775703/4999977c7e1b%2C39971%2C1731217775703.1731217775899 2024-11-10T05:49:35,918 DEBUG [RS:2;4999977c7e1b:39971 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:49:36,295 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-10T05:49:36,295 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T05:49:36,295 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:36,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:36,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:36,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-10T05:49:36,296 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-10T05:49:36,296 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=732657599, stopped=false 2024-11-10T05:49:36,296 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4999977c7e1b,38603,1731217772475 2024-11-10T05:49:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-10T05:49:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:36,298 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:36,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:49:36,298 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-10T05:49:36,298 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:36,299 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:36,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:36,299 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4999977c7e1b,38677,1731217772520' ***** 2024-11-10T05:49:36,299 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T05:49:36,299 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4999977c7e1b,39971,1731217775703' ***** 2024-11-10T05:49:36,299 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-10T05:49:36,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:36,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-10T05:49:36,299 INFO [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T05:49:36,300 INFO [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1739): Disabling cache 2024-11-10T05:49:36,300 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T05:49:36,300 INFO [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence 2024-11-10T05:49:36,300 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T05:49:36,300 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T05:49:36,300 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-10T05:49:36,300 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-10T05:49:36,300 INFO [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T05:49:36,300 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-10T05:49:36,300 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T05:49:36,300 INFO [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e8759e5c-3db5-6a29-1015-63b1e46b29f1/bucket.persistence 2024-11-10T05:49:36,300 INFO [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-10T05:49:36,301 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-10T05:49:36,310 DEBUG [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1488): Thread RS:0;4999977c7e1b:38677 started persisting bucket cache to file 2024-11-10T05:49:36,310 DEBUG [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:36,310 DEBUG [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:36,311 DEBUG [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1488): Thread RS:2;4999977c7e1b:39971 started persisting bucket cache to file 2024-11-10T05:49:36,311 DEBUG [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-10T05:49:36,311 DEBUG [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-10T05:49:36,314 DEBUG [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-10T05:49:36,314 DEBUG [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731217776310 2024-11-10T05:49:36,314 DEBUG [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-10T05:49:36,314 DEBUG [RS:0;4999977c7e1b:38677 {}] bucket.BucketCache(1510): Thread RS:0;4999977c7e1b:38677 finished persisting bucket cache to file, renaming 2024-11-10T05:49:36,314 DEBUG [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731217776311 2024-11-10T05:49:36,314 DEBUG [RS:2;4999977c7e1b:39971 {}] bucket.BucketCache(1510): Thread RS:2;4999977c7e1b:39971 finished persisting bucket cache to file, renaming 2024-11-10T05:49:36,314 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T05:49:36,315 INFO [RS:0;4999977c7e1b:38677 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T05:49:36,315 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T05:49:36,315 INFO [RS:0;4999977c7e1b:38677 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T05:49:36,315 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(3091): Received CLOSE for f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:36,315 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-10T05:49:36,315 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(959): stopping server 4999977c7e1b,38677,1731217772520 2024-11-10T05:49:36,315 INFO [RS:0;4999977c7e1b:38677 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T05:49:36,315 INFO [RS:2;4999977c7e1b:39971 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-10T05:49:36,315 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-10T05:49:36,315 INFO [RS:0;4999977c7e1b:38677 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4999977c7e1b:38677. 2024-11-10T05:49:36,315 INFO [RS:2;4999977c7e1b:39971 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-10T05:49:36,315 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T05:49:36,315 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(959): stopping server 4999977c7e1b,39971,1731217775703 2024-11-10T05:49:36,315 DEBUG [RS:0;4999977c7e1b:38677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:36,315 INFO [RS:2;4999977c7e1b:39971 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T05:49:36,315 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f6aa855be653d4feae71ddefde929d46, disabling compactions & flushes 2024-11-10T05:49:36,316 DEBUG [RS:0;4999977c7e1b:38677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:36,316 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:36,316 INFO [RS:2;4999977c7e1b:39971 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;4999977c7e1b:39971. 2024-11-10T05:49:36,316 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:36,316 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T05:49:36,316 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. after waiting 0 ms 2024-11-10T05:49:36,316 DEBUG [RS:2;4999977c7e1b:39971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-10T05:49:36,316 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T05:49:36,316 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:36,316 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T05:49:36,316 DEBUG [RS:2;4999977c7e1b:39971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:36,316 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-10T05:49:36,316 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(976): stopping server 4999977c7e1b,39971,1731217775703; all regions closed. 2024-11-10T05:49:36,316 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-10T05:49:36,316 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1325): Online Regions={f6aa855be653d4feae71ddefde929d46=testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46., 1588230740=hbase:meta,,1.1588230740} 2024-11-10T05:49:36,316 DEBUG [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, f6aa855be653d4feae71ddefde929d46 2024-11-10T05:49:36,316 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-10T05:49:36,317 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-10T05:49:36,317 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-10T05:49:36,317 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-10T05:49:36,317 DEBUG [StoreFileCloser-f6aa855be653d4feae71ddefde929d46-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/family/d2fb3c87bf0b4ab18a6c18985157bd88 2024-11-10T05:49:36,317 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-10T05:49:36,317 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-10T05:49:36,317 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-10T05:49:36,318 DEBUG [RS:2;4999977c7e1b:39971 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs 2024-11-10T05:49:36,318 INFO [RS:2;4999977c7e1b:39971 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4999977c7e1b%2C39971%2C1731217775703:(num 1731217775899) 2024-11-10T05:49:36,318 DEBUG [RS:2;4999977c7e1b:39971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:36,318 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:36,319 INFO [RS:2;4999977c7e1b:39971 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T05:49:36,319 INFO [RS:2;4999977c7e1b:39971 {}] hbase.ChoreService(370): Chore service for: regionserver/4999977c7e1b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T05:49:36,319 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-10T05:49:36,319 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-10T05:49:36,319 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-10T05:49:36,319 INFO [regionserver/4999977c7e1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T05:49:36,319 INFO [RS:2;4999977c7e1b:39971 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T05:49:36,319 INFO [RS:2;4999977c7e1b:39971 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39971 2024-11-10T05:49:36,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:36,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4999977c7e1b,39971,1731217775703 2024-11-10T05:49:36,321 INFO [RS:2;4999977c7e1b:39971 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T05:49:36,322 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/default/testBlockEvictionOnGracefulStop1731217773496/f6aa855be653d4feae71ddefde929d46/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-10T05:49:36,322 INFO [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:36,323 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f6aa855be653d4feae71ddefde929d46: Waiting for close lock at 1731217776315Running coprocessor pre-close hooks at 1731217776315Disabling compacts and flushes for region at 1731217776315Disabling writes for close at 1731217776316 (+1 ms)Writing region close event to WAL at 1731217776317 (+1 ms)Running coprocessor post-close hooks at 1731217776322 (+5 ms)Closed at 1731217776322 2024-11-10T05:49:36,323 DEBUG [RS_CLOSE_REGION-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46. 2024-11-10T05:49:36,323 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4999977c7e1b,39971,1731217775703] 2024-11-10T05:49:36,324 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4999977c7e1b,39971,1731217775703 already deleted, retry=false 2024-11-10T05:49:36,324 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4999977c7e1b,39971,1731217775703 expired; onlineServers=1 2024-11-10T05:49:36,328 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e132b5f5365447e08e3b0f9e708c03bb 2024-11-10T05:49:36,333 INFO [regionserver/4999977c7e1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:36,343 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/info/e132b5f5365447e08e3b0f9e708c03bb is 209, key is testBlockEvictionOnGracefulStop1731217773496,,1731217773500.f6aa855be653d4feae71ddefde929d46./info:regioninfo/1731217775751/Put/seqid=0 2024-11-10T05:49:36,343 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/info/e132b5f5365447e08e3b0f9e708c03bb 2024-11-10T05:49:36,351 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a57e845ee4f940c18c76c8acae3a603c 2024-11-10T05:49:36,364 INFO [regionserver/4999977c7e1b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:36,365 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/ns/a57e845ee4f940c18c76c8acae3a603c is 43, key is default/ns:d/1731217773466/Put/seqid=0 2024-11-10T05:49:36,366 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/ns/a57e845ee4f940c18c76c8acae3a603c 2024-11-10T05:49:36,374 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bf14933561704b90b3401a7f60df389d 2024-11-10T05:49:36,390 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/table/bf14933561704b90b3401a7f60df389d is 80, key is testBlockEvictionOnGracefulStop1731217773496/table:state/1731217773870/Put/seqid=0 2024-11-10T05:49:36,391 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/table/bf14933561704b90b3401a7f60df389d 2024-11-10T05:49:36,393 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/info/e132b5f5365447e08e3b0f9e708c03bb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/e132b5f5365447e08e3b0f9e708c03bb 2024-11-10T05:49:36,395 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/e132b5f5365447e08e3b0f9e708c03bb, delay=999 ms 2024-11-10T05:49:36,396 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/e132b5f5365447e08e3b0f9e708c03bb, entries=14, sequenceid=13, filesize=7.5 K 2024-11-10T05:49:36,397 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/ns/a57e845ee4f940c18c76c8acae3a603c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/a57e845ee4f940c18c76c8acae3a603c 2024-11-10T05:49:36,399 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/a57e845ee4f940c18c76c8acae3a603c, delay=999 ms 2024-11-10T05:49:36,400 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/a57e845ee4f940c18c76c8acae3a603c, entries=2, sequenceid=13, filesize=5.0 K 2024-11-10T05:49:36,400 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/.tmp/table/bf14933561704b90b3401a7f60df389d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/bf14933561704b90b3401a7f60df389d 2024-11-10T05:49:36,402 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/bf14933561704b90b3401a7f60df389d, delay=991 ms 2024-11-10T05:49:36,403 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/bf14933561704b90b3401a7f60df389d, entries=2, sequenceid=13, filesize=5.3 K 2024-11-10T05:49:36,404 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false 2024-11-10T05:49:36,404 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/info/e132b5f5365447e08e3b0f9e708c03bb 2024-11-10T05:49:36,405 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/ns/a57e845ee4f940c18c76c8acae3a603c 2024-11-10T05:49:36,405 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/table/bf14933561704b90b3401a7f60df389d 2024-11-10T05:49:36,410 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-10T05:49:36,410 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-10T05:49:36,410 INFO [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-10T05:49:36,410 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731217776316Running coprocessor pre-close hooks at 1731217776317 (+1 ms)Disabling compacts and flushes for region at 1731217776317Disabling writes for close at 1731217776317Obtaining lock to block concurrent updates at 1731217776317Preparing flush snapshotting stores in 1588230740 at 1731217776317Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731217776318 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731217776318Flushing 1588230740/info: creating writer at 1731217776318Flushing 1588230740/info: appending metadata at 1731217776342 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731217776342Flushing 1588230740/ns: creating writer at 1731217776345 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731217776365 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1731217776365Flushing 1588230740/table: creating writer at 1731217776368 (+3 ms)Flushing 1588230740/table: appending metadata at 1731217776389 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731217776389Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35cbc8ad: reopening flushed file at 1731217776393 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@227bab84: reopening flushed file at 1731217776397 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5159b3a: reopening flushed file at 1731217776400 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 87ms, sequenceid=13, compaction requested=false at 1731217776404 (+4 ms)Writing region close event to WAL at 1731217776406 (+2 ms)Running coprocessor post-close hooks at 1731217776410 (+4 ms)Closed at 1731217776410 2024-11-10T05:49:36,411 DEBUG [RS_CLOSE_META-regionserver/4999977c7e1b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-10T05:49:36,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:36,423 INFO [RS:2;4999977c7e1b:39971 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T05:49:36,423 INFO [RS:2;4999977c7e1b:39971 {}] regionserver.HRegionServer(1031): Exiting; stopping=4999977c7e1b,39971,1731217775703; zookeeper connection closed. 2024-11-10T05:49:36,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39971-0x10190ba45140003, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:36,424 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@484c0b92 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@484c0b92 2024-11-10T05:49:36,517 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(976): stopping server 4999977c7e1b,38677,1731217772520; all regions closed. 2024-11-10T05:49:36,519 DEBUG [RS:0;4999977c7e1b:38677 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs 2024-11-10T05:49:36,519 INFO [RS:0;4999977c7e1b:38677 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4999977c7e1b%2C38677%2C1731217772520.meta:.meta(num 1731217773419) 2024-11-10T05:49:36,520 DEBUG [RS:0;4999977c7e1b:38677 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/oldWALs 2024-11-10T05:49:36,520 INFO [RS:0;4999977c7e1b:38677 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4999977c7e1b%2C38677%2C1731217772520:(num 1731217773068) 2024-11-10T05:49:36,520 DEBUG [RS:0;4999977c7e1b:38677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-10T05:49:36,520 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.LeaseManager(133): Closed leases 2024-11-10T05:49:36,521 INFO [RS:0;4999977c7e1b:38677 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T05:49:36,521 INFO [RS:0;4999977c7e1b:38677 {}] hbase.ChoreService(370): Chore service for: regionserver/4999977c7e1b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-10T05:49:36,521 INFO [RS:0;4999977c7e1b:38677 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T05:49:36,521 INFO [regionserver/4999977c7e1b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T05:49:36,521 INFO [RS:0;4999977c7e1b:38677 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38677 2024-11-10T05:49:36,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4999977c7e1b,38677,1731217772520 2024-11-10T05:49:36,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-10T05:49:36,523 INFO [RS:0;4999977c7e1b:38677 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T05:49:36,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4999977c7e1b,38677,1731217772520] 2024-11-10T05:49:36,526 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4999977c7e1b,38677,1731217772520 already deleted, retry=false 2024-11-10T05:49:36,526 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4999977c7e1b,38677,1731217772520 expired; onlineServers=0 2024-11-10T05:49:36,526 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4999977c7e1b,38603,1731217772475' ***** 2024-11-10T05:49:36,526 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-10T05:49:36,526 INFO [M:0;4999977c7e1b:38603 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-10T05:49:36,526 INFO [M:0;4999977c7e1b:38603 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-10T05:49:36,526 DEBUG [M:0;4999977c7e1b:38603 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-10T05:49:36,526 DEBUG [M:0;4999977c7e1b:38603 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-10T05:49:36,526 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-10T05:49:36,526 DEBUG [master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.large.0-1731217772802 {}] cleaner.HFileCleaner(306): Exit Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.large.0-1731217772802,5,FailOnTimeoutGroup] 2024-11-10T05:49:36,526 DEBUG [master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.small.0-1731217772802 {}] cleaner.HFileCleaner(306): Exit Thread[master/4999977c7e1b:0:becomeActiveMaster-HFileCleaner.small.0-1731217772802,5,FailOnTimeoutGroup] 2024-11-10T05:49:36,527 INFO [M:0;4999977c7e1b:38603 {}] hbase.ChoreService(370): Chore service for: master/4999977c7e1b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-10T05:49:36,527 INFO [M:0;4999977c7e1b:38603 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-10T05:49:36,527 DEBUG [M:0;4999977c7e1b:38603 {}] master.HMaster(1795): Stopping service threads 2024-11-10T05:49:36,527 INFO [M:0;4999977c7e1b:38603 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:49:36,527 INFO [M:0;4999977c7e1b:38603 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:49:36,527 INFO [M:0;4999977c7e1b:38603 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-10T05:49:36,527 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-10T05:49:36,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-10T05:49:36,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-10T05:49:36,528 DEBUG [M:0;4999977c7e1b:38603 {}] zookeeper.ZKUtil(347): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-10T05:49:36,528 WARN [M:0;4999977c7e1b:38603 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-10T05:49:36,529 INFO [M:0;4999977c7e1b:38603 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/.lastflushedseqids 2024-11-10T05:49:36,536 INFO [M:0;4999977c7e1b:38603 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:49:36,536 INFO [M:0;4999977c7e1b:38603 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-10T05:49:36,536 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:49:36,536 INFO [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:36,536 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:36,536 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:49:36,536 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:36,536 INFO [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-10T05:49:36,546 DEBUG [M:0;4999977c7e1b:38603 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c43a627b2d214f64992875256cccdcaf 2024-11-10T05:49:36,560 DEBUG [M:0;4999977c7e1b:38603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c43a627b2d214f64992875256cccdcaf is 82, key is hbase:meta,,1/info:regioninfo/1731217773452/Put/seqid=0 2024-11-10T05:49:36,561 INFO [M:0;4999977c7e1b:38603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c43a627b2d214f64992875256cccdcaf 2024-11-10T05:49:36,569 DEBUG [M:0;4999977c7e1b:38603 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 48f4e2c23abd48b3ae468f0ed6d37350 2024-11-10T05:49:36,582 DEBUG [M:0;4999977c7e1b:38603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/48f4e2c23abd48b3ae468f0ed6d37350 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731217773874/Put/seqid=0 2024-11-10T05:49:36,583 INFO [M:0;4999977c7e1b:38603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/48f4e2c23abd48b3ae468f0ed6d37350 2024-11-10T05:49:36,591 DEBUG [M:0;4999977c7e1b:38603 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 74aa38853a0d4b7b9012d7282cc1dc9f 2024-11-10T05:49:36,604 DEBUG [M:0;4999977c7e1b:38603 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74aa38853a0d4b7b9012d7282cc1dc9f is 69, key is 4999977c7e1b,38677,1731217772520/rs:state/1731217772905/Put/seqid=0 2024-11-10T05:49:36,605 INFO [M:0;4999977c7e1b:38603 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74aa38853a0d4b7b9012d7282cc1dc9f 2024-11-10T05:49:36,607 INFO [M:0;4999977c7e1b:38603 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74aa38853a0d4b7b9012d7282cc1dc9f 2024-11-10T05:49:36,607 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c43a627b2d214f64992875256cccdcaf as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c43a627b2d214f64992875256cccdcaf 2024-11-10T05:49:36,609 INFO [M:0;4999977c7e1b:38603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c43a627b2d214f64992875256cccdcaf, entries=8, sequenceid=83, filesize=5.5 K 2024-11-10T05:49:36,609 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/48f4e2c23abd48b3ae468f0ed6d37350 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/48f4e2c23abd48b3ae468f0ed6d37350 2024-11-10T05:49:36,611 INFO [M:0;4999977c7e1b:38603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/48f4e2c23abd48b3ae468f0ed6d37350, entries=9, sequenceid=83, filesize=6.4 K 2024-11-10T05:49:36,611 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/74aa38853a0d4b7b9012d7282cc1dc9f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74aa38853a0d4b7b9012d7282cc1dc9f 2024-11-10T05:49:36,613 INFO [M:0;4999977c7e1b:38603 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 74aa38853a0d4b7b9012d7282cc1dc9f 2024-11-10T05:49:36,613 INFO [M:0;4999977c7e1b:38603 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/6c63d340-cb5f-0754-ee8d-323b93b0da80/52113928-04c3-54d4-d62b-1238041c21cd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/74aa38853a0d4b7b9012d7282cc1dc9f, entries=3, sequenceid=83, filesize=5.3 K 2024-11-10T05:49:36,614 INFO [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false 2024-11-10T05:49:36,615 INFO [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:49:36,615 DEBUG [M:0;4999977c7e1b:38603 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731217776536Disabling compacts and flushes for region at 1731217776536Disabling writes for close at 1731217776536Obtaining lock to block concurrent updates at 1731217776536Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731217776536Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1731217776537 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731217776537Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731217776537Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731217776560 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731217776560Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731217776562 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731217776582 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731217776582Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731217776585 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731217776604 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731217776604Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b4e97f4: reopening flushed file at 1731217776607 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76ae3c8c: reopening flushed file at 1731217776609 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ca9805: reopening flushed file at 1731217776611 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=83, compaction requested=false at 1731217776614 (+3 ms)Writing region close event to WAL at 1731217776615 (+1 ms)Closed at 1731217776615 2024-11-10T05:49:36,617 INFO [M:0;4999977c7e1b:38603 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-10T05:49:36,617 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-10T05:49:36,617 INFO [M:0;4999977c7e1b:38603 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38603 2024-11-10T05:49:36,617 INFO [M:0;4999977c7e1b:38603 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-10T05:49:36,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:36,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38677-0x10190ba45140001, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:36,625 INFO [RS:0;4999977c7e1b:38677 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T05:49:36,625 INFO [RS:0;4999977c7e1b:38677 {}] regionserver.HRegionServer(1031): Exiting; stopping=4999977c7e1b,38677,1731217772520; zookeeper connection closed. 2024-11-10T05:49:36,625 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@75c11087 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@75c11087 2024-11-10T05:49:36,626 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-10T05:49:36,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:36,719 INFO [M:0;4999977c7e1b:38603 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-10T05:49:36,719 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38603-0x10190ba45140000, quorum=127.0.0.1:54565, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-10T05:49:36,722 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-10T05:49:36,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-10T05:49:36,723 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:49:36,730 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 452), ProcessCount=11 (was 11), AvailableMemoryMB=8264 (was 8331)