2024-11-11 13:18:15,544 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-11 13:18:15,562 main DEBUG Took 0.016203 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 13:18:15,562 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 13:18:15,563 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 13:18:15,564 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 13:18:15,566 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,576 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 13:18:15,595 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,597 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,599 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,599 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,600 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,601 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,601 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,602 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,603 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,604 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,604 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,605 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,606 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,606 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,607 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,607 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,608 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,609 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,609 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,610 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,611 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,611 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 13:18:15,612 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,613 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 13:18:15,615 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 13:18:15,617 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 13:18:15,619 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 13:18:15,620 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 13:18:15,622 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 13:18:15,623 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 13:18:15,636 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 13:18:15,639 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 13:18:15,641 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 13:18:15,642 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 13:18:15,642 main DEBUG createAppenders(={Console}) 2024-11-11 13:18:15,643 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-11 13:18:15,643 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-11 13:18:15,644 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-11 13:18:15,644 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 13:18:15,644 main DEBUG OutputStream closed 2024-11-11 13:18:15,645 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 13:18:15,645 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 13:18:15,645 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-11 13:18:15,716 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 13:18:15,718 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 13:18:15,719 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 13:18:15,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 13:18:15,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 13:18:15,720 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 13:18:15,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 13:18:15,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 13:18:15,721 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 13:18:15,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 13:18:15,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 13:18:15,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 13:18:15,722 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 13:18:15,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 13:18:15,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 13:18:15,723 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 13:18:15,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 13:18:15,724 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 13:18:15,726 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 13:18:15,727 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-11 13:18:15,727 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 13:18:15,727 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-11T13:18:15,738 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-11 13:18:15,741 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 13:18:15,741 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T13:18:15,977 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142 2024-11-11T13:18:16,004 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=263, ProcessCount=11, AvailableMemoryMB=1087 2024-11-11T13:18:16,126 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T13:18:16,221 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:16,236 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/cluster_180ff733-05ad-b057-42e2-91787d6afc0e, deleteOnExit=true 2024-11-11T13:18:16,322 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/cluster_180ff733-05ad-b057-42e2-91787d6afc0e/zookeeper_0, clientPort=54357, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/cluster_180ff733-05ad-b057-42e2-91787d6afc0e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/cluster_180ff733-05ad-b057-42e2-91787d6afc0e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T13:18:16,337 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54357 2024-11-11T13:18:16,338 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:16,419 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 with version=8 2024-11-11T13:18:16,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/hbase-staging 2024-11-11T13:18:16,522 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T13:18:16,778 INFO [Time-limited test {}] client.ConnectionUtils(128): master/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:18:16,786 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:16,787 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:16,798 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:18:16,798 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:16,799 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:18:16,938 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T13:18:17,021 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:18:17,069 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T13:18:17,079 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T13:18:17,083 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:18:17,139 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 27508 (auto-detected) 2024-11-11T13:18:17,140 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-11T13:18:17,167 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42301 2024-11-11T13:18:17,198 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42301 connecting to ZooKeeper ensemble=127.0.0.1:54357 2024-11-11T13:18:17,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:423010x0, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:18:17,267 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42301-0x1012b55b0d30000 connected 2024-11-11T13:18:17,404 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:17,405 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:17,415 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:17,420 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2, hbase.cluster.distributed=false 2024-11-11T13:18:17,450 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:18:17,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42301 2024-11-11T13:18:17,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42301 2024-11-11T13:18:17,461 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42301 2024-11-11T13:18:17,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42301 2024-11-11T13:18:17,465 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42301 2024-11-11T13:18:17,570 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:18:17,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:17,572 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:17,573 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:18:17,573 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:17,574 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:18:17,578 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T13:18:17,581 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:18:17,582 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:34251 2024-11-11T13:18:17,584 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34251 connecting to ZooKeeper ensemble=127.0.0.1:54357 2024-11-11T13:18:17,585 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:17,586 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:17,599 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:342510x0, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:18:17,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:342510x0, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:17,600 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34251-0x1012b55b0d30001 connected 2024-11-11T13:18:17,605 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T13:18:17,617 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.cache 2024-11-11T13:18:17,623 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T13:18:17,624 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T13:18:17,626 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T13:18:17,628 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T13:18:17,628 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T13:18:17,634 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T13:18:17,639 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T13:18:17,640 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.persistence is complete. 2024-11-11T13:18:17,644 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T13:18:17,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T13:18:17,653 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:18:17,654 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34251 2024-11-11T13:18:17,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34251 2024-11-11T13:18:17,657 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34251 2024-11-11T13:18:17,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34251 2024-11-11T13:18:17,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34251 2024-11-11T13:18:17,681 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:18:17,681 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:17,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:17,682 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:18:17,682 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:17,683 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:18:17,683 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T13:18:17,683 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:18:17,688 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39935 2024-11-11T13:18:17,690 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39935 connecting to ZooKeeper ensemble=127.0.0.1:54357 2024-11-11T13:18:17,691 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:17,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:17,706 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399350x0, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:18:17,707 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:399350x0, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:17,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39935-0x1012b55b0d30002 connected 2024-11-11T13:18:17,708 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T13:18:17,710 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.cache 2024-11-11T13:18:17,711 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T13:18:17,711 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T13:18:17,711 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T13:18:17,715 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T13:18:17,715 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T13:18:17,716 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T13:18:17,716 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.persistence is complete. 2024-11-11T13:18:17,718 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T13:18:17,722 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T13:18:17,723 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T13:18:17,726 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:18:17,730 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39935 2024-11-11T13:18:17,731 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39935 2024-11-11T13:18:17,732 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39935 2024-11-11T13:18:17,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39935 2024-11-11T13:18:17,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39935 2024-11-11T13:18:17,759 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;37d26ecf6460:42301 2024-11-11T13:18:17,760 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/37d26ecf6460,42301,1731331096601 2024-11-11T13:18:17,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:17,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:17,775 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:17,777 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/37d26ecf6460,42301,1731331096601 2024-11-11T13:18:17,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:17,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T13:18:17,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T13:18:17,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:17,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:17,809 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T13:18:17,811 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/37d26ecf6460,42301,1731331096601 from backup master directory 2024-11-11T13:18:17,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:17,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:17,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/37d26ecf6460,42301,1731331096601 2024-11-11T13:18:17,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:17,826 WARN [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:18:17,826 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=37d26ecf6460,42301,1731331096601 2024-11-11T13:18:17,828 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T13:18:17,830 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T13:18:17,839 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/hbase.id] with ID: 6277d374-e58f-41b4-8ba4-8c5cab090c84 2024-11-11T13:18:17,839 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.tmp/hbase.id 2024-11-11T13:18:17,844 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/hbase.id] 2024-11-11T13:18:17,895 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:17,901 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T13:18:17,928 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-11T13:18:17,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:17,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:17,940 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:17,971 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:18:17,975 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T13:18:18,021 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:18:18,026 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:18,079 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store 2024-11-11T13:18:18,100 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T13:18:18,104 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:18,106 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T13:18:18,106 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:18,107 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:18,109 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T13:18:18,109 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:18,110 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:18,111 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731331098106Disabling compacts and flushes for region at 1731331098106Disabling writes for close at 1731331098109 (+3 ms)Writing region close event to WAL at 1731331098110 (+1 ms)Closed at 1731331098110 2024-11-11T13:18:18,111 WARN [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/.initializing 2024-11-11T13:18:18,111 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/WALs/37d26ecf6460,42301,1731331096601 2024-11-11T13:18:18,124 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:18:18,136 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37d26ecf6460%2C42301%2C1731331096601, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/WALs/37d26ecf6460,42301,1731331096601, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/oldWALs, maxLogs=10 2024-11-11T13:18:18,171 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/WALs/37d26ecf6460,42301,1731331096601/37d26ecf6460%2C42301%2C1731331096601.1731331098141 2024-11-11T13:18:18,172 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:18,173 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:18,173 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:18,177 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,178 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T13:18:18,247 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:18,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T13:18:18,254 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:18,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T13:18:18,260 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,261 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:18,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,267 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T13:18:18,267 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,268 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:18,269 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,274 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,275 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,279 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,280 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,284 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T13:18:18,287 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:18,295 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:18:18,296 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74487278, jitterRate=0.10994693636894226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T13:18:18,307 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731331098191Initializing all the Stores at 1731331098191Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331098192 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331098192Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331098193 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331098193Cleaning up temporary data from old regions at 1731331098280 (+87 ms)Region opened successfully at 1731331098307 (+27 ms) 2024-11-11T13:18:18,309 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T13:18:18,349 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcccfc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:18:18,390 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T13:18:18,406 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T13:18:18,407 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T13:18:18,411 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T13:18:18,411 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T13:18:18,419 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-11T13:18:18,419 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T13:18:18,452 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T13:18:18,462 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T13:18:18,473 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T13:18:18,476 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T13:18:18,478 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T13:18:18,489 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T13:18:18,491 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T13:18:18,495 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T13:18:18,505 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T13:18:18,508 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T13:18:18,515 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T13:18:18,533 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T13:18:18,540 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T13:18:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,557 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,560 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=37d26ecf6460,42301,1731331096601, sessionid=0x1012b55b0d30000, setting cluster-up flag (Was=false) 2024-11-11T13:18:18,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,615 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T13:18:18,618 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37d26ecf6460,42301,1731331096601 2024-11-11T13:18:18,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:18,665 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T13:18:18,668 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37d26ecf6460,42301,1731331096601 2024-11-11T13:18:18,672 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T13:18:18,743 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(746): ClusterId : 6277d374-e58f-41b4-8ba4-8c5cab090c84 2024-11-11T13:18:18,743 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(746): ClusterId : 6277d374-e58f-41b4-8ba4-8c5cab090c84 2024-11-11T13:18:18,745 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T13:18:18,745 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T13:18:18,750 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:18,759 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T13:18:18,759 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T13:18:18,759 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T13:18:18,759 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T13:18:18,762 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T13:18:18,766 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T13:18:18,766 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T13:18:18,767 DEBUG [RS:0;37d26ecf6460:34251 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@383ea3e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:18:18,768 DEBUG [RS:1;37d26ecf6460:39935 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c892799, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:18:18,772 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T13:18:18,779 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 37d26ecf6460,42301,1731331096601 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T13:18:18,788 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:18,788 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:18,788 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:18,789 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:18,789 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/37d26ecf6460:0, corePoolSize=10, maxPoolSize=10 2024-11-11T13:18:18,789 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:18,789 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:18:18,789 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:18,791 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731331128791 2024-11-11T13:18:18,792 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T13:18:18,793 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;37d26ecf6460:34251 2024-11-11T13:18:18,793 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;37d26ecf6460:39935 2024-11-11T13:18:18,793 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T13:18:18,794 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:18,794 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T13:18:18,796 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T13:18:18,797 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T13:18:18,797 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T13:18:18,797 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T13:18:18,797 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T13:18:18,797 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T13:18:18,797 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T13:18:18,797 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T13:18:18,797 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T13:18:18,798 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T13:18:18,800 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(2659): reportForDuty to master=37d26ecf6460,42301,1731331096601 with port=34251, startcode=1731331097526 2024-11-11T13:18:18,800 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,800 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T13:18:18,803 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(2659): reportForDuty to master=37d26ecf6460,42301,1731331096601 with port=39935, startcode=1731331097680 2024-11-11T13:18:18,807 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:18,811 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T13:18:18,812 DEBUG [RS:0;37d26ecf6460:34251 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T13:18:18,812 DEBUG [RS:1;37d26ecf6460:39935 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T13:18:18,813 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T13:18:18,813 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T13:18:18,815 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T13:18:18,816 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T13:18:18,817 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731331098817,5,FailOnTimeoutGroup] 2024-11-11T13:18:18,818 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731331098817,5,FailOnTimeoutGroup] 2024-11-11T13:18:18,818 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:18,818 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T13:18:18,819 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:18,819 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:18,822 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T13:18:18,823 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:18,834 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:18,835 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T13:18:18,839 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T13:18:18,839 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:18,840 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T13:18:18,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T13:18:18,844 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:18,844 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T13:18:18,848 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T13:18:18,848 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,848 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:18,849 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T13:18:18,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T13:18:18,859 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:18,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:18,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T13:18:18,860 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:18,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:18,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T13:18:18,861 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T13:18:18,862 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T13:18:18,863 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T13:18:18,879 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:18:18,880 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67240891, jitterRate=0.0019673556089401245}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T13:18:18,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731331098834Initializing all the Stores at 1731331098834Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331098834Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331098835 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331098835Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331098835Cleaning up temporary data from old regions at 1731331098861 (+26 ms)Region opened successfully at 1731331098885 (+24 ms) 2024-11-11T13:18:18,885 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T13:18:18,885 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T13:18:18,885 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53623, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T13:18:18,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53457, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T13:18:18,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T13:18:18,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T13:18:18,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T13:18:18,892 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42301 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37d26ecf6460,34251,1731331097526 2024-11-11T13:18:18,894 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T13:18:18,894 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731331098885Disabling compacts and flushes for region at 1731331098885Disabling writes for close at 1731331098886 (+1 ms)Writing region close event to WAL at 1731331098893 (+7 ms)Closed at 1731331098894 (+1 ms) 2024-11-11T13:18:18,894 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42301 {}] master.ServerManager(517): Registering regionserver=37d26ecf6460,34251,1731331097526 2024-11-11T13:18:18,897 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:18,897 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T13:18:18,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T13:18:18,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42301 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37d26ecf6460,39935,1731331097680 2024-11-11T13:18:18,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42301 {}] master.ServerManager(517): Registering regionserver=37d26ecf6460,39935,1731331097680 2024-11-11T13:18:18,911 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:18,911 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:18,912 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T13:18:18,912 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T13:18:18,912 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T13:18:18,912 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T13:18:18,913 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T13:18:18,916 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T13:18:18,946 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:18:18,948 DEBUG [RS:0;37d26ecf6460:34251 {}] zookeeper.ZKUtil(111): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37d26ecf6460,34251,1731331097526 2024-11-11T13:18:18,948 DEBUG [RS:1;37d26ecf6460:39935 {}] zookeeper.ZKUtil(111): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37d26ecf6460,39935,1731331097680 2024-11-11T13:18:18,948 WARN [RS:0;37d26ecf6460:34251 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:18:18,948 WARN [RS:1;37d26ecf6460:39935 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:18:18,948 INFO [RS:0;37d26ecf6460:34251 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:18,948 INFO [RS:1;37d26ecf6460:39935 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:18,948 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,39935,1731331097680 2024-11-11T13:18:18,948 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,34251,1731331097526 2024-11-11T13:18:18,951 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37d26ecf6460,34251,1731331097526] 2024-11-11T13:18:18,951 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37d26ecf6460,39935,1731331097680] 2024-11-11T13:18:18,979 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T13:18:18,979 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T13:18:19,004 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T13:18:19,004 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T13:18:19,010 INFO [RS:0;37d26ecf6460:34251 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T13:18:19,010 INFO [RS:1;37d26ecf6460:39935 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T13:18:19,010 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,010 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,011 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T13:18:19,011 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T13:18:19,017 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T13:18:19,017 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T13:18:19,019 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,019 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,020 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,020 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:18:19,021 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:18:19,021 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,021 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,022 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,022 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:19,022 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,022 DEBUG [RS:0;37d26ecf6460:34251 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:19,022 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:19,022 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:19,022 DEBUG [RS:1;37d26ecf6460:39935 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:19,028 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,028 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,028 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,028 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,028 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,028 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,39935,1731331097680-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:18:19,030 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,030 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,031 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,031 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,031 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,031 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,34251,1731331097526-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:18:19,059 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T13:18:19,062 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,39935,1731331097680-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,062 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,063 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.Replication(171): 37d26ecf6460,39935,1731331097680 started 2024-11-11T13:18:19,066 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T13:18:19,067 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,34251,1731331097526-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,067 WARN [37d26ecf6460:42301 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T13:18:19,067 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,067 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.Replication(171): 37d26ecf6460,34251,1731331097526 started 2024-11-11T13:18:19,084 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,085 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1482): Serving as 37d26ecf6460,39935,1731331097680, RpcServer on 37d26ecf6460/172.17.0.3:39935, sessionid=0x1012b55b0d30002 2024-11-11T13:18:19,086 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T13:18:19,086 DEBUG [RS:1;37d26ecf6460:39935 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37d26ecf6460,39935,1731331097680 2024-11-11T13:18:19,087 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,39935,1731331097680' 2024-11-11T13:18:19,087 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T13:18:19,088 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T13:18:19,089 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T13:18:19,089 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T13:18:19,089 DEBUG [RS:1;37d26ecf6460:39935 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37d26ecf6460,39935,1731331097680 2024-11-11T13:18:19,089 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,39935,1731331097680' 2024-11-11T13:18:19,089 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T13:18:19,090 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:19,090 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T13:18:19,090 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1482): Serving as 37d26ecf6460,34251,1731331097526, RpcServer on 37d26ecf6460/172.17.0.3:34251, sessionid=0x1012b55b0d30001 2024-11-11T13:18:19,091 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T13:18:19,091 DEBUG [RS:0;37d26ecf6460:34251 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37d26ecf6460,34251,1731331097526 2024-11-11T13:18:19,091 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,34251,1731331097526' 2024-11-11T13:18:19,091 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T13:18:19,091 DEBUG [RS:1;37d26ecf6460:39935 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T13:18:19,091 INFO [RS:1;37d26ecf6460:39935 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T13:18:19,091 INFO [RS:1;37d26ecf6460:39935 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T13:18:19,092 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T13:18:19,092 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T13:18:19,092 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T13:18:19,092 DEBUG [RS:0;37d26ecf6460:34251 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37d26ecf6460,34251,1731331097526 2024-11-11T13:18:19,093 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,34251,1731331097526' 2024-11-11T13:18:19,093 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T13:18:19,093 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T13:18:19,094 DEBUG [RS:0;37d26ecf6460:34251 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T13:18:19,094 INFO [RS:0;37d26ecf6460:34251 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T13:18:19,094 INFO [RS:0;37d26ecf6460:34251 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T13:18:19,199 INFO [RS:0;37d26ecf6460:34251 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:18:19,199 INFO [RS:1;37d26ecf6460:39935 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:18:19,201 INFO [RS:0;37d26ecf6460:34251 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=37d26ecf6460%2C34251%2C1731331097526, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,34251,1731331097526, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs, maxLogs=55 2024-11-11T13:18:19,201 INFO [RS:1;37d26ecf6460:39935 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=37d26ecf6460%2C39935%2C1731331097680, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,39935,1731331097680, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs, maxLogs=55 2024-11-11T13:18:19,222 INFO [RS:0;37d26ecf6460:34251 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,34251,1731331097526/37d26ecf6460%2C34251%2C1731331097526.1731331099204 2024-11-11T13:18:19,223 DEBUG [RS:0;37d26ecf6460:34251 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:19,232 INFO [RS:1;37d26ecf6460:39935 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,39935,1731331097680/37d26ecf6460%2C39935%2C1731331097680.1731331099205 2024-11-11T13:18:19,232 DEBUG [RS:1;37d26ecf6460:39935 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:19,323 DEBUG [37d26ecf6460:42301 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-11T13:18:19,336 DEBUG [37d26ecf6460:42301 {}] balancer.BalancerClusterState(204): Hosts are {37d26ecf6460=0} racks are {/default-rack=0} 2024-11-11T13:18:19,341 DEBUG [37d26ecf6460:42301 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T13:18:19,342 DEBUG [37d26ecf6460:42301 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T13:18:19,342 DEBUG [37d26ecf6460:42301 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T13:18:19,342 DEBUG [37d26ecf6460:42301 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T13:18:19,342 INFO [37d26ecf6460:42301 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T13:18:19,342 INFO [37d26ecf6460:42301 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T13:18:19,342 DEBUG [37d26ecf6460:42301 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T13:18:19,349 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=37d26ecf6460,39935,1731331097680 2024-11-11T13:18:19,354 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37d26ecf6460,39935,1731331097680, state=OPENING 2024-11-11T13:18:19,365 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T13:18:19,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:19,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:19,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:19,375 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:19,375 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:19,375 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:19,378 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T13:18:19,381 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=37d26ecf6460,39935,1731331097680}] 2024-11-11T13:18:19,555 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T13:18:19,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47057, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T13:18:19,568 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T13:18:19,568 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:19,569 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T13:18:19,570 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=37d26ecf6460%2C39935%2C1731331097680.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,39935,1731331097680, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs, maxLogs=55 2024-11-11T13:18:19,592 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,39935,1731331097680/37d26ecf6460%2C39935%2C1731331097680.meta.1731331099573.meta 2024-11-11T13:18:19,593 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:19,593 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:19,595 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T13:18:19,597 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T13:18:19,601 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T13:18:19,604 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T13:18:19,605 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:19,605 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T13:18:19,605 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T13:18:19,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T13:18:19,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T13:18:19,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:19,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:19,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T13:18:19,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T13:18:19,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:19,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:19,610 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T13:18:19,610 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T13:18:19,610 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:19,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:19,611 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T13:18:19,611 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T13:18:19,612 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:19,612 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:19,612 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T13:18:19,613 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:19,614 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:19,614 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T13:18:19,614 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T13:18:19,614 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T13:18:19,615 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T13:18:19,616 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58804060, jitterRate=-0.12375122308731079}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T13:18:19,616 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T13:18:19,618 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731331099606Writing region info on filesystem at 1731331099606Initializing all the Stores at 1731331099606Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331099606Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331099606Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331099607 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331099607Cleaning up temporary data from old regions at 1731331099614 (+7 ms)Running coprocessor post-open hooks at 1731331099616 (+2 ms)Region opened successfully at 1731331099618 (+2 ms) 2024-11-11T13:18:19,624 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731331099545 2024-11-11T13:18:19,633 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T13:18:19,634 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T13:18:19,635 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,39935,1731331097680 2024-11-11T13:18:19,636 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37d26ecf6460,39935,1731331097680, state=OPEN 2024-11-11T13:18:19,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:18:19,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:18:19,929 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:18:19,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:19,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:19,929 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:19,930 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=37d26ecf6460,39935,1731331097680 2024-11-11T13:18:19,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T13:18:19,936 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=37d26ecf6460,39935,1731331097680 in 549 msec 2024-11-11T13:18:19,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T13:18:19,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.0340 sec 2024-11-11T13:18:19,943 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:19,943 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T13:18:19,988 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:18:19,991 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,39935,1731331097680, seqNum=-1] 2024-11-11T13:18:20,023 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:18:20,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60523, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:18:20,054 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3460 sec 2024-11-11T13:18:20,054 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731331100054, completionTime=-1 2024-11-11T13:18:20,058 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-11T13:18:20,059 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T13:18:20,100 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T13:18:20,100 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731331160100 2024-11-11T13:18:20,101 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731331220101 2024-11-11T13:18:20,101 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 41 msec 2024-11-11T13:18:20,103 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-11T13:18:20,111 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,42301,1731331096601-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:20,112 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,42301,1731331096601-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:20,112 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,42301,1731331096601-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:20,114 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-37d26ecf6460:42301, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:20,115 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:20,119 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:20,123 DEBUG [master/37d26ecf6460:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T13:18:20,159 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.333sec 2024-11-11T13:18:20,162 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T13:18:20,163 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T13:18:20,164 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T13:18:20,165 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T13:18:20,165 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T13:18:20,166 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,42301,1731331096601-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:18:20,167 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,42301,1731331096601-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T13:18:20,172 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T13:18:20,173 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T13:18:20,174 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,42301,1731331096601-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:20,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:18:20,275 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T13:18:20,275 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T13:18:20,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,42301,-1 for getting cluster id 2024-11-11T13:18:20,284 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:18:20,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6277d374-e58f-41b4-8ba4-8c5cab090c84' 2024-11-11T13:18:20,301 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:18:20,301 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6277d374-e58f-41b4-8ba4-8c5cab090c84" 2024-11-11T13:18:20,302 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ee5d66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:18:20,302 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,42301,-1] 2024-11-11T13:18:20,305 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:18:20,308 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:20,310 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:18:20,315 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:18:20,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:18:20,325 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,39935,1731331097680, seqNum=-1] 2024-11-11T13:18:20,326 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:18:20,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:18:20,383 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=37d26ecf6460,42301,1731331096601 2024-11-11T13:18:20,389 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:18:20,400 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,42301,1731331096601 2024-11-11T13:18:20,404 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43e5817c 2024-11-11T13:18:20,405 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:18:20,409 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:18:20,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnRegionMove1731331100383', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:18:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 2024-11-11T13:18:20,429 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:18:20,432 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731331100383" procId is: 4 2024-11-11T13:18:20,433 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:20,435 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:18:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:20,461 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => cb633dbba3e492ee2d6a19520c1d3e63, NAME => 'testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731331100383', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:20,468 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:20,469 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(1722): Closing cb633dbba3e492ee2d6a19520c1d3e63, disabling compactions & flushes 2024-11-11T13:18:20,469 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:20,469 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:20,469 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. after waiting 0 ms 2024-11-11T13:18:20,469 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:20,469 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:20,470 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731331100383-pool-0 {}] regionserver.HRegion(1676): Region close journal for cb633dbba3e492ee2d6a19520c1d3e63: Waiting for close lock at 1731331100469Disabling compacts and flushes for region at 1731331100469Disabling writes for close at 1731331100469Writing region close event to WAL at 1731331100469Closed at 1731331100469 2024-11-11T13:18:20,472 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:18:20,478 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731331100472"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731331100472"}]},"ts":"1731331100472"} 2024-11-11T13:18:20,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:18:20,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:18:20,489 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731331100383","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731331100486"}]},"ts":"1731331100486"} 2024-11-11T13:18:20,495 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731331100383, state=ENABLING in hbase:meta 2024-11-11T13:18:20,497 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {37d26ecf6460=0} racks are {/default-rack=0} 2024-11-11T13:18:20,498 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T13:18:20,498 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T13:18:20,498 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T13:18:20,498 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T13:18:20,498 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T13:18:20,498 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T13:18:20,498 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T13:18:20,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, ASSIGN}] 2024-11-11T13:18:20,504 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, ASSIGN 2024-11-11T13:18:20,505 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, ASSIGN; state=OFFLINE, location=37d26ecf6460,39935,1731331097680; forceNewPlan=false, retain=false 2024-11-11T13:18:20,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:20,658 INFO [37d26ecf6460:42301 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T13:18:20,658 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cb633dbba3e492ee2d6a19520c1d3e63, regionState=OPENING, regionLocation=37d26ecf6460,39935,1731331097680 2024-11-11T13:18:20,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, ASSIGN because future has completed 2024-11-11T13:18:20,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,39935,1731331097680}] 2024-11-11T13:18:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:20,822 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:20,822 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => cb633dbba3e492ee2d6a19520c1d3e63, NAME => 'testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:20,823 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731331100383 cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,823 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:20,824 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,824 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,832 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,837 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb633dbba3e492ee2d6a19520c1d3e63 columnFamilyName family 2024-11-11T13:18:20,838 DEBUG [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:20,839 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.HStore(327): Store=cb633dbba3e492ee2d6a19520c1d3e63/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:20,839 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,841 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,841 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,841 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,842 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,843 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,852 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:18:20,853 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened cb633dbba3e492ee2d6a19520c1d3e63; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72173334, jitterRate=0.07546648383140564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T13:18:20,853 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:20,855 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for cb633dbba3e492ee2d6a19520c1d3e63: Running coprocessor pre-open hook at 1731331100824Writing region info on filesystem at 1731331100824Initializing all the Stores at 1731331100825 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331100825Cleaning up temporary data from old regions at 1731331100842 (+17 ms)Running coprocessor post-open hooks at 1731331100854 (+12 ms)Region opened successfully at 1731331100855 (+1 ms) 2024-11-11T13:18:20,857 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63., pid=6, masterSystemTime=1731331100817 2024-11-11T13:18:20,865 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=cb633dbba3e492ee2d6a19520c1d3e63, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,39935,1731331097680 2024-11-11T13:18:20,864 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:20,871 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:20,880 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,39935,1731331097680 because future has completed 2024-11-11T13:18:20,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T13:18:20,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,39935,1731331097680 in 224 msec 2024-11-11T13:18:20,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T13:18:20,900 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, ASSIGN in 394 msec 2024-11-11T13:18:20,902 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:18:20,903 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731331100383","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731331100902"}]},"ts":"1731331100902"} 2024-11-11T13:18:20,912 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731331100383, state=ENABLED in hbase:meta 2024-11-11T13:18:20,914 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:18:20,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731331100383 in 493 msec 2024-11-11T13:18:21,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:21,073 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731331100383 completed 2024-11-11T13:18:21,073 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731331100383 get assigned. Timeout = 60000ms 2024-11-11T13:18:21,076 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T13:18:21,086 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731331100383 assigned to meta. Checking AM states. 2024-11-11T13:18:21,087 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T13:18:21,088 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731331100383 assigned. 2024-11-11T13:18:21,098 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731331100383', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63., hostname=37d26ecf6460,39935,1731331097680, seqNum=2] 2024-11-11T13:18:21,109 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cb633dbba3e492ee2d6a19520c1d3e63 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-11T13:18:21,134 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f91379ba6bcc480da873fc0f016b86dc 2024-11-11T13:18:21,169 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/.tmp/family/f91379ba6bcc480da873fc0f016b86dc is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-11T13:18:21,173 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/.tmp/family/f91379ba6bcc480da873fc0f016b86dc 2024-11-11T13:18:21,203 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/.tmp/family/f91379ba6bcc480da873fc0f016b86dc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc 2024-11-11T13:18:21,204 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc, delay=920 ms 2024-11-11T13:18:21,206 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc, entries=2, sequenceid=6, filesize=4.9 K 2024-11-11T13:18:21,211 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for cb633dbba3e492ee2d6a19520c1d3e63 in 102ms, sequenceid=6, compaction requested=false 2024-11-11T13:18:21,212 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731331100383' 2024-11-11T13:18:21,214 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for cb633dbba3e492ee2d6a19520c1d3e63: 2024-11-11T13:18:22,128 DEBUG [hfile-prefetch-1731331101204 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f91379ba6bcc480da873fc0f016b86dc_0. 2024-11-11T13:18:22,135 DEBUG [hfile-prefetch-1731331101204 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f91379ba6bcc480da873fc0f016b86dc_117. 2024-11-11T13:18:22,136 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc, with total blocks 2, and data blocks 1 2024-11-11T13:18:22,136 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-11T13:18:22,137 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-11T13:18:22,137 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc. Will sleep 100ms and try the verification again. 2024-11-11T13:18:22,219 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T13:18:22,219 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T13:18:22,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:22,220 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:22,230 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T13:18:22,230 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T13:18:22,230 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:22,230 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:22,232 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=cb633dbba3e492ee2d6a19520c1d3e63, source=37d26ecf6460,39935,1731331097680, destination=37d26ecf6460,34251,1731331097526, warming up region on 37d26ecf6460,34251,1731331097526 2024-11-11T13:18:22,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T13:18:22,236 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=cb633dbba3e492ee2d6a19520c1d3e63, source=37d26ecf6460,39935,1731331097680, destination=37d26ecf6460,34251,1731331097526, running balancer 2024-11-11T13:18:22,237 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc, with total blocks 2, and data blocks 1 2024-11-11T13:18:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, REOPEN/MOVE 2024-11-11T13:18:22,238 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2286): iterating over 1 entries in the backing map 2024-11-11T13:18:22,238 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2294): found block for file f91379ba6bcc480da873fc0f016b86dc in the backing map. Acquiring read lock for offset 0 2024-11-11T13:18:22,238 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, REOPEN/MOVE 2024-11-11T13:18:22,238 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2306): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc has now been fully cached. 2024-11-11T13:18:22,238 DEBUG [hfile-prefetch-1731331101204 {}] hfile.PrefetchExecutor(121): Prefetch completed for f91379ba6bcc480da873fc0f016b86dc 2024-11-11T13:18:22,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55577, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T13:18:22,240 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cb633dbba3e492ee2d6a19520c1d3e63, regionState=CLOSING, regionLocation=37d26ecf6460,39935,1731331097680 2024-11-11T13:18:22,243 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(7855): Warmup {ENCODED => cb633dbba3e492ee2d6a19520c1d3e63, NAME => 'testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:22,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:22,249 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T13:18:22,249 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731331102230 2024-11-11T13:18:22,249 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T13:18:22,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, REOPEN/MOVE because future has completed 2024-11-11T13:18:22,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-11T13:18:22,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731331102219 2024-11-11T13:18:22,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T13:18:22,254 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-11T13:18:22,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,39935,1731331097680}] 2024-11-11T13:18:22,255 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,256 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb633dbba3e492ee2d6a19520c1d3e63 columnFamilyName family 2024-11-11T13:18:22,256 DEBUG [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:22,260 DEBUG [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc 2024-11-11T13:18:22,260 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.HStore(327): Store=cb633dbba3e492ee2d6a19520c1d3e63/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(1722): Closing cb633dbba3e492ee2d6a19520c1d3e63, disabling compactions & flushes 2024-11-11T13:18:22,261 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. after waiting 0 ms 2024-11-11T13:18:22,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,265 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34251 {}] regionserver.HRegion(1676): Region close journal for cb633dbba3e492ee2d6a19520c1d3e63: Waiting for close lock at 1731331102260Disabling compacts and flushes for region at 1731331102261 (+1 ms)Disabling writes for close at 1731331102261Writing region close event to WAL at 1731331102265 (+4 ms)Closed at 1731331102265 2024-11-11T13:18:22,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42301 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-11T13:18:22,413 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,413 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-11T13:18:22,414 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing cb633dbba3e492ee2d6a19520c1d3e63, disabling compactions & flushes 2024-11-11T13:18:22,414 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,414 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,414 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. after waiting 0 ms 2024-11-11T13:18:22,414 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,416 DEBUG [StoreFileCloser-cb633dbba3e492ee2d6a19520c1d3e63-family-1 {}] bucket.BucketCache(769): Removing file f91379ba6bcc480da873fc0f016b86dc for region cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,417 DEBUG [StoreFileCloser-cb633dbba3e492ee2d6a19520c1d3e63-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file f91379ba6bcc480da873fc0f016b86dc, starting offset: 0, end offset: 9223372036854775807 2024-11-11T13:18:22,419 DEBUG [StoreFileCloser-cb633dbba3e492ee2d6a19520c1d3e63-family-1 {}] bucket.BucketCache(857): removed key f91379ba6bcc480da873fc0f016b86dc_0 from back map with offset lock 2094080 in the evict process 2024-11-11T13:18:22,424 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-11T13:18:22,425 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,425 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for cb633dbba3e492ee2d6a19520c1d3e63: Waiting for close lock at 1731331102414Running coprocessor pre-close hooks at 1731331102414Disabling compacts and flushes for region at 1731331102414Disabling writes for close at 1731331102414Writing region close event to WAL at 1731331102419 (+5 ms)Running coprocessor post-close hooks at 1731331102424 (+5 ms)Closed at 1731331102425 (+1 ms) 2024-11-11T13:18:22,425 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding cb633dbba3e492ee2d6a19520c1d3e63 move to 37d26ecf6460,34251,1731331097526 record at close sequenceid=6 2024-11-11T13:18:22,429 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,429 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cb633dbba3e492ee2d6a19520c1d3e63, regionState=CLOSED 2024-11-11T13:18:22,433 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,39935,1731331097680 because future has completed 2024-11-11T13:18:22,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T13:18:22,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,39935,1731331097680 in 180 msec 2024-11-11T13:18:22,437 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, REOPEN/MOVE; state=CLOSED, location=37d26ecf6460,34251,1731331097526; forceNewPlan=false, retain=false 2024-11-11T13:18:22,449 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T13:18:22,449 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T13:18:22,450 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:22,450 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:22,452 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T13:18:22,453 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731331102450 2024-11-11T13:18:22,453 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T13:18:22,588 INFO [37d26ecf6460:42301 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T13:18:22,588 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cb633dbba3e492ee2d6a19520c1d3e63, regionState=OPENING, regionLocation=37d26ecf6460,34251,1731331097526 2024-11-11T13:18:22,592 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, REOPEN/MOVE because future has completed 2024-11-11T13:18:22,593 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,34251,1731331097526}] 2024-11-11T13:18:22,750 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,750 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => cb633dbba3e492ee2d6a19520c1d3e63, NAME => 'testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:22,751 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731331100383 cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,752 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:22,752 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,752 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,754 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,756 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb633dbba3e492ee2d6a19520c1d3e63 columnFamilyName family 2024-11-11T13:18:22,756 DEBUG [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:22,761 DEBUG [StoreFileOpener-cb633dbba3e492ee2d6a19520c1d3e63-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc, delay=952 ms 2024-11-11T13:18:22,764 DEBUG [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc 2024-11-11T13:18:22,764 INFO [StoreOpener-cb633dbba3e492ee2d6a19520c1d3e63-1 {}] regionserver.HStore(327): Store=cb633dbba3e492ee2d6a19520c1d3e63/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:22,764 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,765 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,765 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,765 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,766 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,770 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,770 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened cb633dbba3e492ee2d6a19520c1d3e63; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71956503, jitterRate=0.07223545014858246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T13:18:22,771 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,772 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for cb633dbba3e492ee2d6a19520c1d3e63: Running coprocessor pre-open hook at 1731331102752Writing region info on filesystem at 1731331102752Initializing all the Stores at 1731331102753 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331102753Cleaning up temporary data from old regions at 1731331102766 (+13 ms)Running coprocessor post-open hooks at 1731331102771 (+5 ms)Region opened successfully at 1731331102772 (+1 ms) 2024-11-11T13:18:22,772 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63., pid=9, masterSystemTime=1731331102746 2024-11-11T13:18:22,775 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,775 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,776 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=cb633dbba3e492ee2d6a19520c1d3e63, regionState=OPEN, openSeqNum=10, regionLocation=37d26ecf6460,34251,1731331097526 2024-11-11T13:18:22,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,34251,1731331097526 because future has completed 2024-11-11T13:18:22,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-11T13:18:22,782 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure cb633dbba3e492ee2d6a19520c1d3e63, server=37d26ecf6460,34251,1731331097526 in 188 msec 2024-11-11T13:18:22,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731331100383, region=cb633dbba3e492ee2d6a19520c1d3e63, REOPEN/MOVE in 546 msec 2024-11-11T13:18:22,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T13:18:22,791 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:18:22,791 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:22,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:22,796 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:22,796 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:18:22,796 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T13:18:22,796 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1692727640, stopped=false 2024-11-11T13:18:22,797 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=37d26ecf6460,42301,1731331096601 2024-11-11T13:18:22,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:22,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:22,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:22,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:22,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:22,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:22,807 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T13:18:22,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:22,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:22,807 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:22,807 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:18:22,808 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:22,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:22,808 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37d26ecf6460,34251,1731331097526' ***** 2024-11-11T13:18:22,808 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T13:18:22,808 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37d26ecf6460,39935,1731331097680' ***** 2024-11-11T13:18:22,808 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T13:18:22,809 INFO [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T13:18:22,809 INFO [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T13:18:22,809 INFO [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.persistence 2024-11-11T13:18:22,809 INFO [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/bucket.persistence 2024-11-11T13:18:22,809 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T13:18:22,809 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T13:18:22,809 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T13:18:22,809 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T13:18:22,809 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T13:18:22,809 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T13:18:22,809 INFO [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T13:18:22,810 INFO [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T13:18:22,810 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T13:18:22,810 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T13:18:22,820 DEBUG [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1488): Thread RS:0;37d26ecf6460:34251 started persisting bucket cache to file 2024-11-11T13:18:22,820 DEBUG [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1488): Thread RS:1;37d26ecf6460:39935 started persisting bucket cache to file 2024-11-11T13:18:22,820 DEBUG [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:22,820 DEBUG [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:22,820 DEBUG [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:22,820 DEBUG [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:22,822 DEBUG [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T13:18:22,822 DEBUG [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731331102820 2024-11-11T13:18:22,823 DEBUG [RS:0;37d26ecf6460:34251 {}] bucket.BucketCache(1510): Thread RS:0;37d26ecf6460:34251 finished persisting bucket cache to file, renaming 2024-11-11T13:18:22,823 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T13:18:22,823 WARN [Thread-96 {}] util.Shell$1(1036): Error reading the error stream java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell$1.run(Shell.java:1025) ~[hadoop-common-3.4.1.jar:?] 2024-11-11T13:18:22,823 ERROR [RS:1;37d26ecf6460:39935 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1679) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1497) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1775) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T13:18:22,823 INFO [RS:0;37d26ecf6460:34251 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T13:18:22,823 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T13:18:22,823 DEBUG [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T13:18:22,824 INFO [RS:0;37d26ecf6460:34251 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T13:18:22,824 DEBUG [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731331102820 2024-11-11T13:18:22,824 DEBUG [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1510): Thread RS:1;37d26ecf6460:39935 finished persisting bucket cache to file, renaming 2024-11-11T13:18:22,824 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(3091): Received CLOSE for cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,824 WARN [RS:1;37d26ecf6460:39935 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-11T13:18:22,824 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T13:18:22,824 INFO [RS:1;37d26ecf6460:39935 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T13:18:22,824 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T13:18:22,824 INFO [RS:1;37d26ecf6460:39935 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T13:18:22,824 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(959): stopping server 37d26ecf6460,39935,1731331097680 2024-11-11T13:18:22,824 INFO [RS:1;37d26ecf6460:39935 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:18:22,824 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(959): stopping server 37d26ecf6460,34251,1731331097526 2024-11-11T13:18:22,824 INFO [RS:0;37d26ecf6460:34251 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:18:22,824 INFO [RS:1;37d26ecf6460:39935 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;37d26ecf6460:39935. 2024-11-11T13:18:22,824 INFO [RS:0;37d26ecf6460:34251 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;37d26ecf6460:34251. 2024-11-11T13:18:22,824 DEBUG [RS:0;37d26ecf6460:34251 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:22,824 DEBUG [RS:1;37d26ecf6460:39935 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:22,824 DEBUG [RS:0;37d26ecf6460:34251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:22,825 DEBUG [RS:1;37d26ecf6460:39935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:22,825 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T13:18:22,825 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T13:18:22,825 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T13:18:22,825 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1325): Online Regions={cb633dbba3e492ee2d6a19520c1d3e63=testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63.} 2024-11-11T13:18:22,825 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T13:18:22,825 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T13:18:22,825 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T13:18:22,825 DEBUG [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1351): Waiting on cb633dbba3e492ee2d6a19520c1d3e63 2024-11-11T13:18:22,825 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb633dbba3e492ee2d6a19520c1d3e63, disabling compactions & flushes 2024-11-11T13:18:22,825 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,825 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T13:18:22,825 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,825 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T13:18:22,825 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. after waiting 0 ms 2024-11-11T13:18:22,825 DEBUG [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T13:18:22,825 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T13:18:22,826 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,826 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T13:18:22,826 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T13:18:22,826 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T13:18:22,826 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T13:18:22,826 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T13:18:22,826 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-11T13:18:22,826 DEBUG [StoreFileCloser-cb633dbba3e492ee2d6a19520c1d3e63-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/family/f91379ba6bcc480da873fc0f016b86dc 2024-11-11T13:18:22,831 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnRegionMove1731331100383/cb633dbba3e492ee2d6a19520c1d3e63/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-11T13:18:22,831 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,832 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb633dbba3e492ee2d6a19520c1d3e63: Waiting for close lock at 1731331102824Running coprocessor pre-close hooks at 1731331102825 (+1 ms)Disabling compacts and flushes for region at 1731331102825Disabling writes for close at 1731331102825Writing region close event to WAL at 1731331102826 (+1 ms)Running coprocessor post-close hooks at 1731331102831 (+5 ms)Closed at 1731331102831 2024-11-11T13:18:22,832 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63. 2024-11-11T13:18:22,837 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 57ce825df63b4929a440ee767c8c1020 2024-11-11T13:18:22,843 INFO [regionserver/37d26ecf6460:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:22,849 INFO [regionserver/37d26ecf6460:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:22,853 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/info/57ce825df63b4929a440ee767c8c1020 is 205, key is testBlockEvictionOnRegionMove1731331100383,,1731331100411.cb633dbba3e492ee2d6a19520c1d3e63./info:regioninfo/1731331102775/Put/seqid=0 2024-11-11T13:18:22,854 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/info/57ce825df63b4929a440ee767c8c1020 2024-11-11T13:18:22,863 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4abfbfa0ca6c45e49052310c9beca68b 2024-11-11T13:18:22,879 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/ns/4abfbfa0ca6c45e49052310c9beca68b is 43, key is default/ns:d/1731331100033/Put/seqid=0 2024-11-11T13:18:22,880 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/ns/4abfbfa0ca6c45e49052310c9beca68b 2024-11-11T13:18:22,890 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4df938ea617645faba2a1332c26813f8 2024-11-11T13:18:22,903 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/table/4df938ea617645faba2a1332c26813f8 is 78, key is testBlockEvictionOnRegionMove1731331100383/table:state/1731331100902/Put/seqid=0 2024-11-11T13:18:22,903 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/table/4df938ea617645faba2a1332c26813f8 2024-11-11T13:18:22,906 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/info/57ce825df63b4929a440ee767c8c1020 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/57ce825df63b4929a440ee767c8c1020 2024-11-11T13:18:22,907 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/57ce825df63b4929a440ee767c8c1020, delay=997 ms 2024-11-11T13:18:22,908 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/57ce825df63b4929a440ee767c8c1020, entries=15, sequenceid=15, filesize=7.6 K 2024-11-11T13:18:22,908 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/ns/4abfbfa0ca6c45e49052310c9beca68b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/4abfbfa0ca6c45e49052310c9beca68b 2024-11-11T13:18:22,909 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/4abfbfa0ca6c45e49052310c9beca68b, delay=999 ms 2024-11-11T13:18:22,911 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/4abfbfa0ca6c45e49052310c9beca68b, entries=2, sequenceid=15, filesize=5.0 K 2024-11-11T13:18:22,911 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/table/4df938ea617645faba2a1332c26813f8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4df938ea617645faba2a1332c26813f8 2024-11-11T13:18:22,912 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4df938ea617645faba2a1332c26813f8, delay=943 ms 2024-11-11T13:18:22,913 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4df938ea617645faba2a1332c26813f8, entries=2, sequenceid=15, filesize=5.3 K 2024-11-11T13:18:22,914 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=15, compaction requested=false 2024-11-11T13:18:22,914 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T13:18:22,915 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/57ce825df63b4929a440ee767c8c1020 2024-11-11T13:18:22,916 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/4abfbfa0ca6c45e49052310c9beca68b 2024-11-11T13:18:22,917 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4df938ea617645faba2a1332c26813f8 2024-11-11T13:18:22,921 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-11T13:18:22,922 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T13:18:22,922 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T13:18:22,923 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731331102825Running coprocessor pre-close hooks at 1731331102826 (+1 ms)Disabling compacts and flushes for region at 1731331102826Disabling writes for close at 1731331102826Obtaining lock to block concurrent updates at 1731331102826Preparing flush snapshotting stores in 1588230740 at 1731331102826Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731331102827 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731331102827Flushing 1588230740/info: creating writer at 1731331102827Flushing 1588230740/info: appending metadata at 1731331102853 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731331102853Flushing 1588230740/ns: creating writer at 1731331102857 (+4 ms)Flushing 1588230740/ns: appending metadata at 1731331102878 (+21 ms)Flushing 1588230740/ns: closing flushed file at 1731331102878Flushing 1588230740/table: creating writer at 1731331102884 (+6 ms)Flushing 1588230740/table: appending metadata at 1731331102902 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731331102902Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69105dbf: reopening flushed file at 1731331102905 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f00e940: reopening flushed file at 1731331102908 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c6d6a9b: reopening flushed file at 1731331102911 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 88ms, sequenceid=15, compaction requested=false at 1731331102914 (+3 ms)Writing region close event to WAL at 1731331102917 (+3 ms)Running coprocessor post-close hooks at 1731331102921 (+4 ms)Closed at 1731331102922 (+1 ms) 2024-11-11T13:18:22,923 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T13:18:23,026 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(976): stopping server 37d26ecf6460,34251,1731331097526; all regions closed. 2024-11-11T13:18:23,026 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(976): stopping server 37d26ecf6460,39935,1731331097680; all regions closed. 2024-11-11T13:18:23,031 DEBUG [RS:1;37d26ecf6460:39935 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs 2024-11-11T13:18:23,031 DEBUG [RS:0;37d26ecf6460:34251 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs 2024-11-11T13:18:23,031 INFO [RS:1;37d26ecf6460:39935 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C39935%2C1731331097680.meta:.meta(num 1731331099573) 2024-11-11T13:18:23,031 INFO [RS:0;37d26ecf6460:34251 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C34251%2C1731331097526:(num 1731331099204) 2024-11-11T13:18:23,031 DEBUG [RS:0;37d26ecf6460:34251 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:23,031 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:23,031 INFO [RS:0;37d26ecf6460:34251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:18:23,032 INFO [RS:0;37d26ecf6460:34251 {}] hbase.ChoreService(370): Chore service for: regionserver/37d26ecf6460:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T13:18:23,032 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T13:18:23,032 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T13:18:23,032 INFO [regionserver/37d26ecf6460:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:18:23,032 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T13:18:23,032 INFO [RS:0;37d26ecf6460:34251 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:18:23,033 INFO [RS:0;37d26ecf6460:34251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:34251 2024-11-11T13:18:23,033 DEBUG [RS:1;37d26ecf6460:39935 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs 2024-11-11T13:18:23,033 INFO [RS:1;37d26ecf6460:39935 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C39935%2C1731331097680:(num 1731331099205) 2024-11-11T13:18:23,033 DEBUG [RS:1;37d26ecf6460:39935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:23,033 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:23,033 INFO [RS:1;37d26ecf6460:39935 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:18:23,033 INFO [RS:1;37d26ecf6460:39935 {}] hbase.ChoreService(370): Chore service for: regionserver/37d26ecf6460:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-11T13:18:23,034 INFO [RS:1;37d26ecf6460:39935 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:18:23,034 INFO [regionserver/37d26ecf6460:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:18:23,034 INFO [RS:1;37d26ecf6460:39935 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39935 2024-11-11T13:18:23,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:18:23,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37d26ecf6460,39935,1731331097680 2024-11-11T13:18:23,047 INFO [RS:1;37d26ecf6460:39935 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:18:23,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37d26ecf6460,34251,1731331097526 2024-11-11T13:18:23,047 INFO [RS:0;37d26ecf6460:34251 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:18:23,057 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37d26ecf6460,34251,1731331097526] 2024-11-11T13:18:23,065 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37d26ecf6460,34251,1731331097526 already deleted, retry=false 2024-11-11T13:18:23,065 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 37d26ecf6460,34251,1731331097526 expired; onlineServers=1 2024-11-11T13:18:23,065 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37d26ecf6460,39935,1731331097680] 2024-11-11T13:18:23,073 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37d26ecf6460,39935,1731331097680 already deleted, retry=false 2024-11-11T13:18:23,073 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 37d26ecf6460,39935,1731331097680 expired; onlineServers=0 2024-11-11T13:18:23,073 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '37d26ecf6460,42301,1731331096601' ***** 2024-11-11T13:18:23,073 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T13:18:23,074 INFO [M:0;37d26ecf6460:42301 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:18:23,074 INFO [M:0;37d26ecf6460:42301 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:18:23,074 DEBUG [M:0;37d26ecf6460:42301 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T13:18:23,074 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T13:18:23,074 DEBUG [M:0;37d26ecf6460:42301 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T13:18:23,074 DEBUG [master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731331098817 {}] cleaner.HFileCleaner(306): Exit Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731331098817,5,FailOnTimeoutGroup] 2024-11-11T13:18:23,074 DEBUG [master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731331098817 {}] cleaner.HFileCleaner(306): Exit Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731331098817,5,FailOnTimeoutGroup] 2024-11-11T13:18:23,074 INFO [M:0;37d26ecf6460:42301 {}] hbase.ChoreService(370): Chore service for: master/37d26ecf6460:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T13:18:23,074 INFO [M:0;37d26ecf6460:42301 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:18:23,074 DEBUG [M:0;37d26ecf6460:42301 {}] master.HMaster(1795): Stopping service threads 2024-11-11T13:18:23,075 INFO [M:0;37d26ecf6460:42301 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T13:18:23,075 INFO [M:0;37d26ecf6460:42301 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T13:18:23,075 INFO [M:0;37d26ecf6460:42301 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T13:18:23,075 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T13:18:23,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T13:18:23,081 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,082 DEBUG [M:0;37d26ecf6460:42301 {}] zookeeper.ZKUtil(347): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T13:18:23,082 WARN [M:0;37d26ecf6460:42301 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T13:18:23,082 INFO [M:0;37d26ecf6460:42301 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.lastflushedseqids 2024-11-11T13:18:23,090 INFO [M:0;37d26ecf6460:42301 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T13:18:23,091 INFO [M:0;37d26ecf6460:42301 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T13:18:23,091 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T13:18:23,091 INFO [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,091 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,091 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T13:18:23,091 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,091 INFO [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-11T13:18:23,100 DEBUG [M:0;37d26ecf6460:42301 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 910ca0c12ee7438f9934c92415edc3a1 2024-11-11T13:18:23,112 DEBUG [M:0;37d26ecf6460:42301 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/910ca0c12ee7438f9934c92415edc3a1 is 82, key is hbase:meta,,1/info:regioninfo/1731331099634/Put/seqid=0 2024-11-11T13:18:23,113 INFO [M:0;37d26ecf6460:42301 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/910ca0c12ee7438f9934c92415edc3a1 2024-11-11T13:18:23,120 DEBUG [M:0;37d26ecf6460:42301 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d76a12d44a0d46cca86f1f2f7a47a973 2024-11-11T13:18:23,132 DEBUG [M:0;37d26ecf6460:42301 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d76a12d44a0d46cca86f1f2f7a47a973 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731331100916/Put/seqid=0 2024-11-11T13:18:23,133 INFO [M:0;37d26ecf6460:42301 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d76a12d44a0d46cca86f1f2f7a47a973 2024-11-11T13:18:23,141 DEBUG [M:0;37d26ecf6460:42301 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 35f334662ff04979a4cd406e516c1784 2024-11-11T13:18:23,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:23,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:23,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39935-0x1012b55b0d30002, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:23,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34251-0x1012b55b0d30001, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:23,157 INFO [RS:1;37d26ecf6460:39935 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:18:23,157 INFO [RS:0;37d26ecf6460:34251 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:18:23,158 INFO [RS:0;37d26ecf6460:34251 {}] regionserver.HRegionServer(1031): Exiting; stopping=37d26ecf6460,34251,1731331097526; zookeeper connection closed. 2024-11-11T13:18:23,158 INFO [RS:1;37d26ecf6460:39935 {}] regionserver.HRegionServer(1031): Exiting; stopping=37d26ecf6460,39935,1731331097680; zookeeper connection closed. 2024-11-11T13:18:23,158 DEBUG [M:0;37d26ecf6460:42301 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/35f334662ff04979a4cd406e516c1784 is 69, key is 37d26ecf6460,34251,1731331097526/rs:state/1731331098897/Put/seqid=0 2024-11-11T13:18:23,158 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@65fa096 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@65fa096 2024-11-11T13:18:23,158 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c5076aa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c5076aa 2024-11-11T13:18:23,159 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-11T13:18:23,159 INFO [M:0;37d26ecf6460:42301 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/35f334662ff04979a4cd406e516c1784 2024-11-11T13:18:23,161 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/910ca0c12ee7438f9934c92415edc3a1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/910ca0c12ee7438f9934c92415edc3a1 2024-11-11T13:18:23,164 INFO [M:0;37d26ecf6460:42301 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/910ca0c12ee7438f9934c92415edc3a1, entries=8, sequenceid=84, filesize=5.5 K 2024-11-11T13:18:23,164 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/d76a12d44a0d46cca86f1f2f7a47a973 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d76a12d44a0d46cca86f1f2f7a47a973 2024-11-11T13:18:23,167 INFO [M:0;37d26ecf6460:42301 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/d76a12d44a0d46cca86f1f2f7a47a973, entries=9, sequenceid=84, filesize=6.5 K 2024-11-11T13:18:23,167 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/35f334662ff04979a4cd406e516c1784 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/35f334662ff04979a4cd406e516c1784 2024-11-11T13:18:23,170 INFO [M:0;37d26ecf6460:42301 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/35f334662ff04979a4cd406e516c1784, entries=2, sequenceid=84, filesize=5.1 K 2024-11-11T13:18:23,171 INFO [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=84, compaction requested=false 2024-11-11T13:18:23,172 INFO [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,173 DEBUG [M:0;37d26ecf6460:42301 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731331103091Disabling compacts and flushes for region at 1731331103091Disabling writes for close at 1731331103091Obtaining lock to block concurrent updates at 1731331103091Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731331103091Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731331103092 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731331103092Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731331103092Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731331103111 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731331103112 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731331103114 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731331103132 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731331103132Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731331103134 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731331103157 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731331103157Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d25118: reopening flushed file at 1731331103160 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a8d19f4: reopening flushed file at 1731331103164 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5aa0f3b9: reopening flushed file at 1731331103167 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 79ms, sequenceid=84, compaction requested=false at 1731331103171 (+4 ms)Writing region close event to WAL at 1731331103172 (+1 ms)Closed at 1731331103172 2024-11-11T13:18:23,174 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:18:23,174 INFO [M:0;37d26ecf6460:42301 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T13:18:23,174 INFO [M:0;37d26ecf6460:42301 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42301 2024-11-11T13:18:23,175 INFO [M:0;37d26ecf6460:42301 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:18:23,281 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:23,281 INFO [M:0;37d26ecf6460:42301 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:18:23,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42301-0x1012b55b0d30000, quorum=127.0.0.1:54357, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:23,286 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T13:18:23,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T13:18:23,306 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,313 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/37d26ecf6460:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/37d26ecf6460:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731331102761 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731331101204 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731331102909 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/37d26ecf6460:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731331102907 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/37d26ecf6460:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=260 (was 263), ProcessCount=11 (was 11), AvailableMemoryMB=550 (was 1087) 2024-11-11T13:18:23,318 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=260, ProcessCount=11, AvailableMemoryMB=549 2024-11-11T13:18:23,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/hadoop-log-dir so I do NOT create it in target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2 2024-11-11T13:18:23,318 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/hadoop-tmp-dir so I do NOT create it in target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2 2024-11-11T13:18:23,319 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,321 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/cluster_a2aa70f7-45c5-2159-ee8c-b4da9666b8d9, deleteOnExit=true 2024-11-11T13:18:23,323 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/cluster_a2aa70f7-45c5-2159-ee8c-b4da9666b8d9/zookeeper_0, clientPort=49804, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/cluster_a2aa70f7-45c5-2159-ee8c-b4da9666b8d9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/cluster_a2aa70f7-45c5-2159-ee8c-b4da9666b8d9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T13:18:23,324 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49804 2024-11-11T13:18:23,336 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 with version=8 2024-11-11T13:18:23,336 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/hbase-staging 2024-11-11T13:18:23,339 INFO [Time-limited test {}] client.ConnectionUtils(128): master/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:18:23,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,339 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:18:23,339 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,340 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:18:23,340 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T13:18:23,340 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:18:23,341 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37611 2024-11-11T13:18:23,343 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37611 connecting to ZooKeeper ensemble=127.0.0.1:49804 2024-11-11T13:18:23,401 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:376110x0, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:18:23,402 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37611-0x1012b55cc430000 connected 2024-11-11T13:18:23,478 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,478 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,480 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:23,480 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2, hbase.cluster.distributed=false 2024-11-11T13:18:23,482 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:18:23,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37611 2024-11-11T13:18:23,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37611 2024-11-11T13:18:23,482 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37611 2024-11-11T13:18:23,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37611 2024-11-11T13:18:23,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37611 2024-11-11T13:18:23,498 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:18:23,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,498 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,499 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:18:23,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,499 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:18:23,499 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T13:18:23,499 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:18:23,500 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36877 2024-11-11T13:18:23,502 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36877 connecting to ZooKeeper ensemble=127.0.0.1:49804 2024-11-11T13:18:23,502 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,503 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,511 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:368770x0, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:18:23,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36877-0x1012b55cc430001 connected 2024-11-11T13:18:23,512 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:23,512 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T13:18:23,513 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.cache 2024-11-11T13:18:23,513 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T13:18:23,513 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T13:18:23,513 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T13:18:23,518 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T13:18:23,518 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T13:18:23,518 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T13:18:23,519 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T13:18:23,519 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T13:18:23,519 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence is complete. 2024-11-11T13:18:23,520 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T13:18:23,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:18:23,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36877 2024-11-11T13:18:23,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36877 2024-11-11T13:18:23,527 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36877 2024-11-11T13:18:23,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36877 2024-11-11T13:18:23,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36877 2024-11-11T13:18:23,553 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:18:23,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,553 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:18:23,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:23,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:18:23,554 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T13:18:23,554 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:18:23,555 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44079 2024-11-11T13:18:23,557 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44079 connecting to ZooKeeper ensemble=127.0.0.1:49804 2024-11-11T13:18:23,558 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,558 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,577 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440790x0, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:18:23,577 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44079-0x1012b55cc430002 connected 2024-11-11T13:18:23,577 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:23,578 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T13:18:23,582 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.cache 2024-11-11T13:18:23,582 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T13:18:23,582 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T13:18:23,583 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T13:18:23,583 INFO [Thread-120 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T13:18:23,583 WARN [Thread-120 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-11T13:18:23,583 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T13:18:23,583 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T13:18:23,583 INFO [Thread-120 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence is complete. 2024-11-11T13:18:23,587 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T13:18:23,588 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T13:18:23,590 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:18:23,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44079 2024-11-11T13:18:23,594 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44079 2024-11-11T13:18:23,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44079 2024-11-11T13:18:23,595 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44079 2024-11-11T13:18:23,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44079 2024-11-11T13:18:23,606 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;37d26ecf6460:37611 2024-11-11T13:18:23,606 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/37d26ecf6460,37611,1731331103338 2024-11-11T13:18:23,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:23,611 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:23,611 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:23,612 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/37d26ecf6460,37611,1731331103338 2024-11-11T13:18:23,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T13:18:23,619 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T13:18:23,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,620 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,621 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T13:18:23,621 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/37d26ecf6460,37611,1731331103338 from backup master directory 2024-11-11T13:18:23,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:23,628 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:23,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/37d26ecf6460,37611,1731331103338 2024-11-11T13:18:23,628 WARN [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:18:23,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T13:18:23,628 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=37d26ecf6460,37611,1731331103338 2024-11-11T13:18:23,629 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/hbase.id] with ID: 37df1686-b9cf-4e50-a88c-251056fee634 2024-11-11T13:18:23,630 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.tmp/hbase.id 2024-11-11T13:18:23,633 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/hbase.id] 2024-11-11T13:18:23,666 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:23,666 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T13:18:23,669 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-11T13:18:23,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,678 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,683 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:18:23,685 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T13:18:23,685 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:23,706 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store 2024-11-11T13:18:23,713 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:23,713 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T13:18:23,713 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,713 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,713 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T13:18:23,713 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,713 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:23,714 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731331103713Disabling compacts and flushes for region at 1731331103713Disabling writes for close at 1731331103713Writing region close event to WAL at 1731331103713Closed at 1731331103713 2024-11-11T13:18:23,714 WARN [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/.initializing 2024-11-11T13:18:23,714 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/WALs/37d26ecf6460,37611,1731331103338 2024-11-11T13:18:23,721 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:18:23,726 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=37d26ecf6460%2C37611%2C1731331103338, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/WALs/37d26ecf6460,37611,1731331103338, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/oldWALs, maxLogs=10 2024-11-11T13:18:23,743 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/WALs/37d26ecf6460,37611,1731331103338/37d26ecf6460%2C37611%2C1731331103338.1731331103727 2024-11-11T13:18:23,743 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:23,743 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:23,744 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:23,744 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,744 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T13:18:23,749 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:23,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,754 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T13:18:23,754 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:23,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T13:18:23,759 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:23,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T13:18:23,763 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:23,763 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,764 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,764 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,764 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,764 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,764 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T13:18:23,765 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T13:18:23,770 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:18:23,771 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70611438, jitterRate=0.05219241976737976}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T13:18:23,772 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731331103744Initializing all the Stores at 1731331103744Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331103744Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331103744Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331103744Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331103744Cleaning up temporary data from old regions at 1731331103764 (+20 ms)Region opened successfully at 1731331103772 (+8 ms) 2024-11-11T13:18:23,773 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T13:18:23,777 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74c496b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:18:23,778 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T13:18:23,778 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T13:18:23,778 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T13:18:23,779 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T13:18:23,779 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-11T13:18:23,780 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-11T13:18:23,780 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T13:18:23,782 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T13:18:23,783 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T13:18:23,793 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T13:18:23,794 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T13:18:23,795 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T13:18:23,803 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T13:18:23,803 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T13:18:23,805 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T13:18:23,811 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T13:18:23,812 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T13:18:23,819 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T13:18:23,822 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T13:18:23,836 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T13:18:23,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:23,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:23,853 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:23,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,853 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,854 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=37d26ecf6460,37611,1731331103338, sessionid=0x1012b55cc430000, setting cluster-up flag (Was=false) 2024-11-11T13:18:23,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,870 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,895 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T13:18:23,896 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37d26ecf6460,37611,1731331103338 2024-11-11T13:18:23,911 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:23,936 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T13:18:23,937 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=37d26ecf6460,37611,1731331103338 2024-11-11T13:18:23,938 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T13:18:23,940 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:23,940 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T13:18:23,940 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T13:18:23,941 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 37d26ecf6460,37611,1731331103338 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T13:18:23,942 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:23,943 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:23,943 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:23,943 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/37d26ecf6460:0, corePoolSize=5, maxPoolSize=5 2024-11-11T13:18:23,943 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/37d26ecf6460:0, corePoolSize=10, maxPoolSize=10 2024-11-11T13:18:23,943 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:23,943 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:18:23,943 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:23,944 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731331133944 2024-11-11T13:18:23,944 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T13:18:23,944 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T13:18:23,944 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T13:18:23,944 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T13:18:23,944 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T13:18:23,944 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T13:18:23,945 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:23,945 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:23,945 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T13:18:23,945 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T13:18:23,945 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T13:18:23,945 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T13:18:23,945 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,946 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T13:18:23,946 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T13:18:23,946 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731331103946,5,FailOnTimeoutGroup] 2024-11-11T13:18:23,946 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T13:18:23,946 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731331103946,5,FailOnTimeoutGroup] 2024-11-11T13:18:23,946 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:23,946 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T13:18:23,946 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:23,946 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:23,962 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T13:18:23,962 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:23,968 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:23,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T13:18:23,972 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T13:18:23,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:23,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T13:18:23,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T13:18:23,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:23,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T13:18:23,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T13:18:23,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:23,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T13:18:23,981 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T13:18:23,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:23,982 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:23,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T13:18:23,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:23,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:23,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T13:18:23,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T13:18:23,983 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T13:18:23,984 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T13:18:23,989 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:18:23,990 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74895316, jitterRate=0.11602717638015747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T13:18:23,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731331103968Initializing all the Stores at 1731331103968Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331103968Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331103968Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331103968Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331103968Cleaning up temporary data from old regions at 1731331103983 (+15 ms)Region opened successfully at 1731331103990 (+7 ms) 2024-11-11T13:18:23,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T13:18:23,991 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T13:18:23,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T13:18:23,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T13:18:23,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T13:18:23,992 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T13:18:23,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731331103991Disabling compacts and flushes for region at 1731331103991Disabling writes for close at 1731331103991Writing region close event to WAL at 1731331103991Closed at 1731331103991 2024-11-11T13:18:23,993 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:23,993 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T13:18:23,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T13:18:23,995 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T13:18:23,996 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T13:18:23,998 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(746): ClusterId : 37df1686-b9cf-4e50-a88c-251056fee634 2024-11-11T13:18:23,998 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(746): ClusterId : 37df1686-b9cf-4e50-a88c-251056fee634 2024-11-11T13:18:23,998 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T13:18:23,998 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T13:18:24,003 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T13:18:24,003 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T13:18:24,004 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T13:18:24,004 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T13:18:24,012 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T13:18:24,012 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T13:18:24,013 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@690ccba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:18:24,013 DEBUG [RS:1;37d26ecf6460:44079 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b650caf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:18:24,025 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;37d26ecf6460:44079 2024-11-11T13:18:24,025 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T13:18:24,025 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T13:18:24,026 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T13:18:24,027 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(2659): reportForDuty to master=37d26ecf6460,37611,1731331103338 with port=44079, startcode=1731331103552 2024-11-11T13:18:24,027 DEBUG [RS:1;37d26ecf6460:44079 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T13:18:24,035 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45165, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T13:18:24,035 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;37d26ecf6460:36877 2024-11-11T13:18:24,035 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T13:18:24,035 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T13:18:24,035 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T13:18:24,035 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37611 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,035 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37611 {}] master.ServerManager(517): Registering regionserver=37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,036 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(2659): reportForDuty to master=37d26ecf6460,37611,1731331103338 with port=36877, startcode=1731331103498 2024-11-11T13:18:24,036 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T13:18:24,037 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:24,037 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T13:18:24,037 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T13:18:24,038 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40949, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T13:18:24,039 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37611 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37d26ecf6460,36877,1731331103498 2024-11-11T13:18:24,039 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37611 {}] master.ServerManager(517): Registering regionserver=37d26ecf6460,36877,1731331103498 2024-11-11T13:18:24,041 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:24,041 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T13:18:24,041 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T13:18:24,044 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:18:24,090 DEBUG [RS:1;37d26ecf6460:44079 {}] zookeeper.ZKUtil(111): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,090 WARN [RS:1;37d26ecf6460:44079 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:18:24,090 INFO [RS:1;37d26ecf6460:44079 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:24,090 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37d26ecf6460,36877,1731331103498] 2024-11-11T13:18:24,090 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37d26ecf6460,44079,1731331103552] 2024-11-11T13:18:24,090 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,090 DEBUG [RS:0;37d26ecf6460:36877 {}] zookeeper.ZKUtil(111): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37d26ecf6460,36877,1731331103498 2024-11-11T13:18:24,090 WARN [RS:0;37d26ecf6460:36877 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:18:24,090 INFO [RS:0;37d26ecf6460:36877 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:24,091 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,36877,1731331103498 2024-11-11T13:18:24,096 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T13:18:24,099 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T13:18:24,100 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T13:18:24,100 INFO [RS:1;37d26ecf6460:44079 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T13:18:24,100 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,102 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T13:18:24,102 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T13:18:24,103 INFO [RS:0;37d26ecf6460:36877 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T13:18:24,103 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T13:18:24,103 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,103 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,103 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,104 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,105 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:24,105 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 DEBUG [RS:1;37d26ecf6460:44079 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:24,105 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,105 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,106 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:18:24,106 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,106 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,106 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,106 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,106 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,106 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,106 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,106 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,44079,1731331103552-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:18:24,106 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,106 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,106 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:24,107 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:24,107 DEBUG [RS:0;37d26ecf6460:36877 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:24,109 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,110 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,110 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,110 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,110 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,110 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,36877,1731331103498-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:18:24,124 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T13:18:24,124 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,36877,1731331103498-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,125 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,125 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.Replication(171): 37d26ecf6460,36877,1731331103498 started 2024-11-11T13:18:24,127 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T13:18:24,128 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,44079,1731331103552-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,128 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,128 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.Replication(171): 37d26ecf6460,44079,1731331103552 started 2024-11-11T13:18:24,140 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,140 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1482): Serving as 37d26ecf6460,36877,1731331103498, RpcServer on 37d26ecf6460/172.17.0.3:36877, sessionid=0x1012b55cc430001 2024-11-11T13:18:24,140 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T13:18:24,140 DEBUG [RS:0;37d26ecf6460:36877 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37d26ecf6460,36877,1731331103498 2024-11-11T13:18:24,140 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,36877,1731331103498' 2024-11-11T13:18:24,140 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T13:18:24,141 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T13:18:24,142 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T13:18:24,142 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T13:18:24,142 DEBUG [RS:0;37d26ecf6460:36877 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37d26ecf6460,36877,1731331103498 2024-11-11T13:18:24,142 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,36877,1731331103498' 2024-11-11T13:18:24,142 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T13:18:24,143 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T13:18:24,143 DEBUG [RS:0;37d26ecf6460:36877 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T13:18:24,143 INFO [RS:0;37d26ecf6460:36877 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T13:18:24,143 INFO [RS:0;37d26ecf6460:36877 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T13:18:24,146 WARN [37d26ecf6460:37611 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T13:18:24,149 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,149 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1482): Serving as 37d26ecf6460,44079,1731331103552, RpcServer on 37d26ecf6460/172.17.0.3:44079, sessionid=0x1012b55cc430002 2024-11-11T13:18:24,150 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T13:18:24,150 DEBUG [RS:1;37d26ecf6460:44079 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,150 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,44079,1731331103552' 2024-11-11T13:18:24,150 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T13:18:24,151 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T13:18:24,151 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T13:18:24,151 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T13:18:24,151 DEBUG [RS:1;37d26ecf6460:44079 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,151 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,44079,1731331103552' 2024-11-11T13:18:24,151 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T13:18:24,152 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T13:18:24,152 DEBUG [RS:1;37d26ecf6460:44079 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T13:18:24,152 INFO [RS:1;37d26ecf6460:44079 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T13:18:24,152 INFO [RS:1;37d26ecf6460:44079 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T13:18:24,244 INFO [RS:0;37d26ecf6460:36877 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:18:24,245 INFO [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=37d26ecf6460%2C36877%2C1731331103498, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,36877,1731331103498, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs, maxLogs=55 2024-11-11T13:18:24,253 INFO [RS:1;37d26ecf6460:44079 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:18:24,254 INFO [RS:1;37d26ecf6460:44079 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=37d26ecf6460%2C44079%2C1731331103552, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,44079,1731331103552, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs, maxLogs=55 2024-11-11T13:18:24,272 INFO [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,36877,1731331103498/37d26ecf6460%2C36877%2C1731331103498.1731331104246 2024-11-11T13:18:24,272 DEBUG [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:24,275 INFO [RS:1;37d26ecf6460:44079 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,44079,1731331103552/37d26ecf6460%2C44079%2C1731331103552.1731331104256 2024-11-11T13:18:24,276 DEBUG [RS:1;37d26ecf6460:44079 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:24,396 DEBUG [37d26ecf6460:37611 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-11T13:18:24,397 DEBUG [37d26ecf6460:37611 {}] balancer.BalancerClusterState(204): Hosts are {37d26ecf6460=0} racks are {/default-rack=0} 2024-11-11T13:18:24,398 DEBUG [37d26ecf6460:37611 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T13:18:24,398 DEBUG [37d26ecf6460:37611 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T13:18:24,398 DEBUG [37d26ecf6460:37611 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T13:18:24,398 DEBUG [37d26ecf6460:37611 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T13:18:24,398 INFO [37d26ecf6460:37611 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T13:18:24,398 INFO [37d26ecf6460:37611 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T13:18:24,398 DEBUG [37d26ecf6460:37611 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T13:18:24,399 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,400 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37d26ecf6460,44079,1731331103552, state=OPENING 2024-11-11T13:18:24,410 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T13:18:24,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:24,420 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:24,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:24,420 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T13:18:24,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:24,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:24,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=37d26ecf6460,44079,1731331103552}] 2024-11-11T13:18:24,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:24,574 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T13:18:24,577 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53747, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T13:18:24,580 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T13:18:24,580 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:24,581 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T13:18:24,582 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=37d26ecf6460%2C44079%2C1731331103552.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,44079,1731331103552, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs, maxLogs=55 2024-11-11T13:18:24,601 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,44079,1731331103552/37d26ecf6460%2C44079%2C1731331103552.meta.1731331104582.meta 2024-11-11T13:18:24,602 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:24,602 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:24,603 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T13:18:24,603 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T13:18:24,603 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T13:18:24,603 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T13:18:24,604 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:24,604 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T13:18:24,604 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T13:18:24,605 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-11T13:18:24,606 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T13:18:24,606 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:24,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:24,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-11T13:18:24,607 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T13:18:24,607 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:24,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:24,607 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-11T13:18:24,608 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T13:18:24,608 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:24,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:24,608 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-11T13:18:24,609 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T13:18:24,609 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:24,609 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T13:18:24,609 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T13:18:24,610 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:24,611 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740 2024-11-11T13:18:24,611 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T13:18:24,611 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T13:18:24,611 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T13:18:24,613 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T13:18:24,613 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70735424, jitterRate=0.054039955139160156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T13:18:24,613 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T13:18:24,614 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731331104604Writing region info on filesystem at 1731331104604Initializing all the Stores at 1731331104604Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331104605 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331104605Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331104605Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731331104605Cleaning up temporary data from old regions at 1731331104611 (+6 ms)Running coprocessor post-open hooks at 1731331104613 (+2 ms)Region opened successfully at 1731331104614 (+1 ms) 2024-11-11T13:18:24,615 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731331104573 2024-11-11T13:18:24,619 DEBUG [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T13:18:24,619 INFO [RS_OPEN_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T13:18:24,620 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,621 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 37d26ecf6460,44079,1731331103552, state=OPEN 2024-11-11T13:18:24,628 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:18:24,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:18:24,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T13:18:24,628 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=37d26ecf6460,44079,1731331103552 2024-11-11T13:18:24,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:24,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:24,628 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T13:18:24,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T13:18:24,631 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=37d26ecf6460,44079,1731331103552 in 208 msec 2024-11-11T13:18:24,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T13:18:24,634 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 638 msec 2024-11-11T13:18:24,635 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T13:18:24,635 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T13:18:24,638 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:18:24,638 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,44079,1731331103552, seqNum=-1] 2024-11-11T13:18:24,639 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:18:24,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38165, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:18:24,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 707 msec 2024-11-11T13:18:24,651 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731331104651, completionTime=-1 2024-11-11T13:18:24,651 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-11T13:18:24,652 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T13:18:24,656 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-11T13:18:24,656 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731331164656 2024-11-11T13:18:24,656 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731331224656 2024-11-11T13:18:24,656 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-11T13:18:24,657 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,37611,1731331103338-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,657 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,37611,1731331103338-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,657 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,37611,1731331103338-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,657 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-37d26ecf6460:37611, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,657 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,659 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,661 DEBUG [master/37d26ecf6460:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T13:18:24,666 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.038sec 2024-11-11T13:18:24,667 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T13:18:24,667 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T13:18:24,667 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T13:18:24,667 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T13:18:24,667 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T13:18:24,667 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,37611,1731331103338-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:18:24,667 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,37611,1731331103338-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T13:18:24,668 DEBUG [master/37d26ecf6460:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T13:18:24,668 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T13:18:24,669 INFO [master/37d26ecf6460:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,37611,1731331103338-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:24,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:18:24,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 37d26ecf6460,37611,-1 for getting cluster id 2024-11-11T13:18:24,700 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T13:18:24,703 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '37df1686-b9cf-4e50-a88c-251056fee634' 2024-11-11T13:18:24,703 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T13:18:24,704 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "37df1686-b9cf-4e50-a88c-251056fee634" 2024-11-11T13:18:24,711 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e6fdbcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:18:24,711 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [37d26ecf6460,37611,-1] 2024-11-11T13:18:24,712 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T13:18:24,715 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:24,722 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T13:18:24,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T13:18:24,725 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T13:18:24,727 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=37d26ecf6460,44079,1731331103552, seqNum=-1] 2024-11-11T13:18:24,728 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:18:24,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:18:24,738 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=37d26ecf6460,37611,1731331103338 2024-11-11T13:18:24,739 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T13:18:24,744 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 37d26ecf6460,37611,1731331103338 2024-11-11T13:18:24,744 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@62d06445 2024-11-11T13:18:24,744 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T13:18:24,755 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T13:18:24,756 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37611 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnGracefulStop1731331104738', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T13:18:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37611 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 2024-11-11T13:18:24,759 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T13:18:24,760 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:24,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37611 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731331104738" procId is: 4 2024-11-11T13:18:24,761 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T13:18:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:24,782 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ddb4e4f7d13ad045e0af24a7d77999d6, NAME => 'testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731331104738', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:24,807 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:24,807 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(1722): Closing ddb4e4f7d13ad045e0af24a7d77999d6, disabling compactions & flushes 2024-11-11T13:18:24,807 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:24,807 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:24,807 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. after waiting 0 ms 2024-11-11T13:18:24,808 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:24,808 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:24,808 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731331104738-pool-0 {}] regionserver.HRegion(1676): Region close journal for ddb4e4f7d13ad045e0af24a7d77999d6: Waiting for close lock at 1731331104807Disabling compacts and flushes for region at 1731331104807Disabling writes for close at 1731331104808 (+1 ms)Writing region close event to WAL at 1731331104808Closed at 1731331104808 2024-11-11T13:18:24,810 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T13:18:24,811 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731331104811"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731331104811"}]},"ts":"1731331104811"} 2024-11-11T13:18:24,815 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T13:18:24,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T13:18:24,818 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731331104738","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731331104818"}]},"ts":"1731331104818"} 2024-11-11T13:18:24,823 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731331104738, state=ENABLING in hbase:meta 2024-11-11T13:18:24,823 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {37d26ecf6460=0} racks are {/default-rack=0} 2024-11-11T13:18:24,824 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-11T13:18:24,824 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-11T13:18:24,824 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-11T13:18:24,824 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-11T13:18:24,824 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-11T13:18:24,824 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-11T13:18:24,824 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-11T13:18:24,825 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN}] 2024-11-11T13:18:24,827 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN 2024-11-11T13:18:24,828 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN; state=OFFLINE, location=37d26ecf6460,36877,1731331103498; forceNewPlan=false, retain=false 2024-11-11T13:18:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:24,979 INFO [37d26ecf6460:37611 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-11T13:18:24,980 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ddb4e4f7d13ad045e0af24a7d77999d6, regionState=OPENING, regionLocation=37d26ecf6460,36877,1731331103498 2024-11-11T13:18:24,985 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN because future has completed 2024-11-11T13:18:24,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddb4e4f7d13ad045e0af24a7d77999d6, server=37d26ecf6460,36877,1731331103498}] 2024-11-11T13:18:25,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:25,107 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T13:18:25,138 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T13:18:25,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59559, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T13:18:25,143 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:25,143 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ddb4e4f7d13ad045e0af24a7d77999d6, NAME => 'testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:25,144 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731331104738 ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,144 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:25,144 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,144 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,145 INFO [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,150 INFO [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddb4e4f7d13ad045e0af24a7d77999d6 columnFamilyName family 2024-11-11T13:18:25,150 DEBUG [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:25,151 INFO [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] regionserver.HStore(327): Store=ddb4e4f7d13ad045e0af24a7d77999d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:25,151 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,152 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,152 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,153 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,153 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,154 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,162 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T13:18:25,162 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ddb4e4f7d13ad045e0af24a7d77999d6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64900191, jitterRate=-0.032911792397499084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T13:18:25,163 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:25,163 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ddb4e4f7d13ad045e0af24a7d77999d6: Running coprocessor pre-open hook at 1731331105144Writing region info on filesystem at 1731331105144Initializing all the Stores at 1731331105145 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331105145Cleaning up temporary data from old regions at 1731331105153 (+8 ms)Running coprocessor post-open hooks at 1731331105163 (+10 ms)Region opened successfully at 1731331105163 2024-11-11T13:18:25,164 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6., pid=6, masterSystemTime=1731331105138 2024-11-11T13:18:25,168 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:25,168 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:25,168 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ddb4e4f7d13ad045e0af24a7d77999d6, regionState=OPEN, openSeqNum=2, regionLocation=37d26ecf6460,36877,1731331103498 2024-11-11T13:18:25,172 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddb4e4f7d13ad045e0af24a7d77999d6, server=37d26ecf6460,36877,1731331103498 because future has completed 2024-11-11T13:18:25,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T13:18:25,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ddb4e4f7d13ad045e0af24a7d77999d6, server=37d26ecf6460,36877,1731331103498 in 188 msec 2024-11-11T13:18:25,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T13:18:25,178 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN in 351 msec 2024-11-11T13:18:25,180 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T13:18:25,180 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731331104738","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731331105180"}]},"ts":"1731331105180"} 2024-11-11T13:18:25,183 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731331104738, state=ENABLED in hbase:meta 2024-11-11T13:18:25,184 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T13:18:25,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731331104738 in 428 msec 2024-11-11T13:18:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37611 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T13:18:25,392 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731331104738 completed 2024-11-11T13:18:25,392 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731331104738 get assigned. Timeout = 60000ms 2024-11-11T13:18:25,393 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T13:18:25,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731331104738 assigned to meta. Checking AM states. 2024-11-11T13:18:25,397 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-11T13:18:25,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731331104738 assigned. 2024-11-11T13:18:25,401 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731331104738', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6., hostname=37d26ecf6460,36877,1731331103498, seqNum=2] 2024-11-11T13:18:25,401 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T13:18:25,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T13:18:25,409 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ddb4e4f7d13ad045e0af24a7d77999d6 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-11T13:18:25,419 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 98289c58989c4690b089a1ca3eb22d71 2024-11-11T13:18:25,432 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/.tmp/family/98289c58989c4690b089a1ca3eb22d71 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-11T13:18:25,433 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/.tmp/family/98289c58989c4690b089a1ca3eb22d71 2024-11-11T13:18:25,435 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/.tmp/family/98289c58989c4690b089a1ca3eb22d71 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71 2024-11-11T13:18:25,436 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71, delay=998 ms 2024-11-11T13:18:25,438 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71, entries=2, sequenceid=6, filesize=4.9 K 2024-11-11T13:18:25,438 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for ddb4e4f7d13ad045e0af24a7d77999d6 in 29ms, sequenceid=6, compaction requested=false 2024-11-11T13:18:25,438 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731331104738' 2024-11-11T13:18:25,439 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ddb4e4f7d13ad045e0af24a7d77999d6: 2024-11-11T13:18:26,436 DEBUG [hfile-prefetch-1731331101204 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 98289c58989c4690b089a1ca3eb22d71_0. 2024-11-11T13:18:26,437 DEBUG [hfile-prefetch-1731331101204 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 98289c58989c4690b089a1ca3eb22d71_117. 2024-11-11T13:18:26,437 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71, with total blocks 2, and data blocks 1 2024-11-11T13:18:26,437 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-11T13:18:26,437 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-11T13:18:26,438 DEBUG [hfile-prefetch-1731331101204 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71. Will sleep 100ms and try the verification again. 2024-11-11T13:18:26,440 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;37d26ecf6460:36877,5,FailOnTimeoutGroup] 2024-11-11T13:18:26,440 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37d26ecf6460,36877,1731331103498' ***** 2024-11-11T13:18:26,440 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-11T13:18:26,440 INFO [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T13:18:26,475 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T13:18:26,475 INFO [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence 2024-11-11T13:18:26,475 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T13:18:26,475 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T13:18:26,475 INFO [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T13:18:26,475 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T13:18:26,485 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-11T13:18:26,485 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-11T13:18:26,485 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:26,485 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:26,485 DEBUG [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1488): Thread RS:0;37d26ecf6460:36877 started persisting bucket cache to file 2024-11-11T13:18:26,486 DEBUG [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:26,486 DEBUG [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:26,489 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T13:18:26,489 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731331106485 2024-11-11T13:18:26,489 DEBUG [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-11T13:18:26,489 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-11T13:18:26,489 DEBUG [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731331106486 2024-11-11T13:18:26,489 DEBUG [RS:0;37d26ecf6460:36877 {}] bucket.BucketCache(1510): Thread RS:0;37d26ecf6460:36877 finished persisting bucket cache to file, renaming 2024-11-11T13:18:26,489 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T13:18:26,489 INFO [RS:0;37d26ecf6460:36877 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T13:18:26,489 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T13:18:26,489 INFO [RS:0;37d26ecf6460:36877 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T13:18:26,489 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(3091): Received CLOSE for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:26,490 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(959): stopping server 37d26ecf6460,36877,1731331103498 2024-11-11T13:18:26,490 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:18:26,490 INFO [RS:0;37d26ecf6460:36877 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;37d26ecf6460:36877. 2024-11-11T13:18:26,490 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T13:18:26,490 DEBUG [RS:0;37d26ecf6460:36877 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:26,490 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:26,490 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ddb4e4f7d13ad045e0af24a7d77999d6, disabling compactions & flushes 2024-11-11T13:18:26,490 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T13:18:26,490 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:26,490 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1325): Online Regions={ddb4e4f7d13ad045e0af24a7d77999d6=testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.} 2024-11-11T13:18:26,490 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:26,490 DEBUG [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1351): Waiting on ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:26,491 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. after waiting 0 ms 2024-11-11T13:18:26,491 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:26,491 DEBUG [StoreFileCloser-ddb4e4f7d13ad045e0af24a7d77999d6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71 2024-11-11T13:18:26,494 WARN [hfile-prefetch-1731331101204 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-11T13:18:26,495 DEBUG [hfile-prefetch-1731331101204 {}] hfile.PrefetchExecutor(121): Prefetch completed for 98289c58989c4690b089a1ca3eb22d71 2024-11-11T13:18:26,497 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-11T13:18:26,497 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:26,498 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ddb4e4f7d13ad045e0af24a7d77999d6: Waiting for close lock at 1731331106490Running coprocessor pre-close hooks at 1731331106490Disabling compacts and flushes for region at 1731331106490Disabling writes for close at 1731331106491 (+1 ms)Writing region close event to WAL at 1731331106492 (+1 ms)Running coprocessor post-close hooks at 1731331106497 (+5 ms)Closed at 1731331106497 2024-11-11T13:18:26,498 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:26,512 INFO [regionserver/37d26ecf6460:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:26,691 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(976): stopping server 37d26ecf6460,36877,1731331103498; all regions closed. 2024-11-11T13:18:26,693 DEBUG [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs 2024-11-11T13:18:26,693 INFO [RS:0;37d26ecf6460:36877 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C36877%2C1731331103498:(num 1731331104246) 2024-11-11T13:18:26,693 DEBUG [RS:0;37d26ecf6460:36877 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:26,693 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:26,693 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:18:26,694 INFO [RS:0;37d26ecf6460:36877 {}] hbase.ChoreService(370): Chore service for: regionserver/37d26ecf6460:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T13:18:26,694 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T13:18:26,694 INFO [regionserver/37d26ecf6460:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:18:26,694 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T13:18:26,694 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T13:18:26,694 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:18:26,694 INFO [RS:0;37d26ecf6460:36877 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36877 2024-11-11T13:18:26,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:18:26,710 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37d26ecf6460,36877,1731331103498 2024-11-11T13:18:26,710 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:18:26,719 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37d26ecf6460,36877,1731331103498] 2024-11-11T13:18:26,727 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37d26ecf6460,36877,1731331103498 already deleted, retry=false 2024-11-11T13:18:26,728 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 37d26ecf6460,36877,1731331103498 on 37d26ecf6460,37611,1731331103338 2024-11-11T13:18:26,733 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 37d26ecf6460,36877,1731331103498, splitWal=true, meta=false 2024-11-11T13:18:26,736 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 37d26ecf6460,36877,1731331103498, splitWal=true, meta=false 2024-11-11T13:18:26,736 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 37d26ecf6460,36877,1731331103498 (carryingMeta=false) 37d26ecf6460,36877,1731331103498/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@4c8af9dc[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-11T13:18:26,737 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 37d26ecf6460,36877,1731331103498 had 1 regions 2024-11-11T13:18:26,738 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 37d26ecf6460,36877,1731331103498, splitWal=true, meta=false, isMeta: false 2024-11-11T13:18:26,738 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,36877,1731331103498-splitting 2024-11-11T13:18:26,739 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,36877,1731331103498-splitting dir is empty, no logs to split. 2024-11-11T13:18:26,739 INFO [PEWorker-2 {}] master.SplitWALManager(105): 37d26ecf6460,36877,1731331103498 WAL count=0, meta=false 2024-11-11T13:18:26,741 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,36877,1731331103498-splitting dir is empty, no logs to split. 2024-11-11T13:18:26,741 INFO [PEWorker-2 {}] master.SplitWALManager(105): 37d26ecf6460,36877,1731331103498 WAL count=0, meta=false 2024-11-11T13:18:26,741 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 37d26ecf6460,36877,1731331103498 WAL splitting is done? wals=0, meta=false 2024-11-11T13:18:26,741 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 37d26ecf6460,36877,1731331103498 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,36877,1731331103498-splitting does not exist 2024-11-11T13:18:26,743 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN}] 2024-11-11T13:18:26,744 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN 2024-11-11T13:18:26,745 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-11T13:18:26,820 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:26,820 INFO [RS:0;37d26ecf6460:36877 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:18:26,820 INFO [RS:0;37d26ecf6460:36877 {}] regionserver.HRegionServer(1031): Exiting; stopping=37d26ecf6460,36877,1731331103498; zookeeper connection closed. 2024-11-11T13:18:26,820 DEBUG [pool-163-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36877-0x1012b55cc430001, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:26,821 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1aae6289 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1aae6289 2024-11-11T13:18:26,896 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ddb4e4f7d13ad045e0af24a7d77999d6, regionState=OPENING, regionLocation=37d26ecf6460,44079,1731331103552 2024-11-11T13:18:26,899 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN because future has completed 2024-11-11T13:18:26,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddb4e4f7d13ad045e0af24a7d77999d6, server=37d26ecf6460,44079,1731331103552}] 2024-11-11T13:18:26,962 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/37d26ecf6460:0 server-side Connection retries=45 2024-11-11T13:18:26,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:26,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:26,962 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T13:18:26,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T13:18:26,962 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T13:18:26,962 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T13:18:26,962 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T13:18:26,963 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33597 2024-11-11T13:18:26,964 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33597 connecting to ZooKeeper ensemble=127.0.0.1:49804 2024-11-11T13:18:26,965 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:26,965 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:26,976 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335970x0, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T13:18:26,977 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-11T13:18:26,977 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33597-0x1012b55cc430003 connected 2024-11-11T13:18:26,977 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-11T13:18:26,978 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T13:18:26,979 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.cache 2024-11-11T13:18:26,979 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-11T13:18:26,979 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-11T13:18:26,979 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-11T13:18:26,979 INFO [Thread-200 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-11T13:18:26,979 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-11T13:18:26,980 INFO [Thread-200 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-11T13:18:26,980 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T13:18:26,981 INFO [Thread-200 {}] bucket.BucketCache(1706): Retrieved 1 of chunks with blockCount = 1. 2024-11-11T13:18:26,981 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T13:18:26,983 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T13:18:26,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33597 2024-11-11T13:18:26,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33597 2024-11-11T13:18:26,986 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33597 2024-11-11T13:18:26,986 WARN [Thread-200 {}] bucket.BucketCache(1614): Checksum for cache file failed. We need to validate each cache key in the backing map. This may take some time, so we'll do it in a background thread, 2024-11-11T13:18:26,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33597 2024-11-11T13:18:26,987 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33597 2024-11-11T13:18:26,987 DEBUG [Thread-200 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-11T13:18:26,988 INFO [Thread-200 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-11T13:18:26,988 INFO [Thread-200 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 1 2024-11-11T13:18:26,988 INFO [Thread-200 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence is complete. 2024-11-11T13:18:26,988 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(746): ClusterId : 37df1686-b9cf-4e50-a88c-251056fee634 2024-11-11T13:18:26,989 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T13:18:26,995 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T13:18:26,995 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T13:18:27,003 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T13:18:27,004 DEBUG [RS:2;37d26ecf6460:33597 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c97e571, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=37d26ecf6460/172.17.0.3:0 2024-11-11T13:18:27,017 DEBUG [RS:2;37d26ecf6460:33597 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;37d26ecf6460:33597 2024-11-11T13:18:27,017 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T13:18:27,017 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T13:18:27,017 DEBUG [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T13:18:27,018 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(2659): reportForDuty to master=37d26ecf6460,37611,1731331103338 with port=33597, startcode=1731331106961 2024-11-11T13:18:27,018 DEBUG [RS:2;37d26ecf6460:33597 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T13:18:27,021 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53813, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T13:18:27,021 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37611 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,021 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37611 {}] master.ServerManager(517): Registering regionserver=37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,023 DEBUG [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2 2024-11-11T13:18:27,023 DEBUG [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-11T13:18:27,023 DEBUG [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T13:18:27,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:18:27,028 DEBUG [RS:2;37d26ecf6460:33597 {}] zookeeper.ZKUtil(111): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,028 WARN [RS:2;37d26ecf6460:33597 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T13:18:27,028 INFO [RS:2;37d26ecf6460:33597 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T13:18:27,028 DEBUG [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,028 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [37d26ecf6460,33597,1731331106961] 2024-11-11T13:18:27,033 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T13:18:27,036 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T13:18:27,036 INFO [RS:2;37d26ecf6460:33597 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T13:18:27,036 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,036 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T13:18:27,037 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T13:18:27,037 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,037 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,037 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,037 INFO [Thread-202 {}] bucket.BucketCache(1638): Finished validating 1 keys in the backing map. Recovered: 1. This took 0ms. 2024-11-11T13:18:27,037 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,037 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/37d26ecf6460:0, corePoolSize=2, maxPoolSize=2 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/37d26ecf6460:0, corePoolSize=1, maxPoolSize=1 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:27,038 DEBUG [RS:2;37d26ecf6460:33597 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/37d26ecf6460:0, corePoolSize=3, maxPoolSize=3 2024-11-11T13:18:27,039 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,039 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,039 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,039 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,039 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,039 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,33597,1731331106961-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T13:18:27,055 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,056 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => ddb4e4f7d13ad045e0af24a7d77999d6, NAME => 'testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.', STARTKEY => '', ENDKEY => ''} 2024-11-11T13:18:27,056 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T13:18:27,056 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731331104738 ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,057 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=37d26ecf6460,33597,1731331106961-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,057 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T13:18:27,057 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,057 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.Replication(171): 37d26ecf6460,33597,1731331106961 started 2024-11-11T13:18:27,057 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,057 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,058 INFO [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,059 INFO [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ddb4e4f7d13ad045e0af24a7d77999d6 columnFamilyName family 2024-11-11T13:18:27,059 DEBUG [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T13:18:27,060 DEBUG [StoreFileOpener-ddb4e4f7d13ad045e0af24a7d77999d6-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71, delay=926 ms 2024-11-11T13:18:27,061 DEBUG [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71 2024-11-11T13:18:27,062 INFO [StoreOpener-ddb4e4f7d13ad045e0af24a7d77999d6-1 {}] regionserver.HStore(327): Store=ddb4e4f7d13ad045e0af24a7d77999d6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T13:18:27,062 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,063 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T13:18:27,063 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,063 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T13:18:27,064 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,064 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,064 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T13:18:27,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T13:18:27,065 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T13:18:27,065 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T13:18:27,066 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,066 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened ddb4e4f7d13ad045e0af24a7d77999d6; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62560276, jitterRate=-0.06777924299240112}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T13:18:27,066 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,067 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for ddb4e4f7d13ad045e0af24a7d77999d6: Running coprocessor pre-open hook at 1731331107057Writing region info on filesystem at 1731331107057Initializing all the Stores at 1731331107058 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731331107058Cleaning up temporary data from old regions at 1731331107064 (+6 ms)Running coprocessor post-open hooks at 1731331107066 (+2 ms)Region opened successfully at 1731331107067 (+1 ms) 2024-11-11T13:18:27,068 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6., pid=9, masterSystemTime=1731331107053 2024-11-11T13:18:27,071 DEBUG [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,071 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=ddb4e4f7d13ad045e0af24a7d77999d6, regionState=OPEN, openSeqNum=10, regionLocation=37d26ecf6460,44079,1731331103552 2024-11-11T13:18:27,071 INFO [RS_OPEN_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,074 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure ddb4e4f7d13ad045e0af24a7d77999d6, server=37d26ecf6460,44079,1731331103552 because future has completed 2024-11-11T13:18:27,074 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T13:18:27,074 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(1482): Serving as 37d26ecf6460,33597,1731331106961, RpcServer on 37d26ecf6460/172.17.0.3:33597, sessionid=0x1012b55cc430003 2024-11-11T13:18:27,074 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T13:18:27,074 DEBUG [RS:2;37d26ecf6460:33597 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,074 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,33597,1731331106961' 2024-11-11T13:18:27,075 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T13:18:27,075 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T13:18:27,076 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T13:18:27,076 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T13:18:27,076 DEBUG [RS:2;37d26ecf6460:33597 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,076 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '37d26ecf6460,33597,1731331106961' 2024-11-11T13:18:27,076 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T13:18:27,077 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T13:18:27,077 DEBUG [RS:2;37d26ecf6460:33597 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T13:18:27,077 INFO [RS:2;37d26ecf6460:33597 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T13:18:27,077 INFO [RS:2;37d26ecf6460:33597 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T13:18:27,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-11T13:18:27,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure ddb4e4f7d13ad045e0af24a7d77999d6, server=37d26ecf6460,44079,1731331103552 in 176 msec 2024-11-11T13:18:27,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T13:18:27,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731331104738, region=ddb4e4f7d13ad045e0af24a7d77999d6, ASSIGN in 335 msec 2024-11-11T13:18:27,079 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 37d26ecf6460,36877,1731331103498 after splitting done 2024-11-11T13:18:27,081 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 37d26ecf6460,36877,1731331103498, splitWal=true, meta=false in 351 msec 2024-11-11T13:18:27,179 INFO [RS:2;37d26ecf6460:33597 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T13:18:27,182 INFO [RS:2;37d26ecf6460:33597 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=37d26ecf6460%2C33597%2C1731331106961, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,33597,1731331106961, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs, maxLogs=55 2024-11-11T13:18:27,208 INFO [RS:2;37d26ecf6460:33597 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/WALs/37d26ecf6460,33597,1731331106961/37d26ecf6460%2C33597%2C1731331106961.1731331107185 2024-11-11T13:18:27,209 DEBUG [RS:2;37d26ecf6460:33597 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-11T13:18:27,575 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T13:18:27,575 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:18:27,575 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:27,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:27,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:27,576 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T13:18:27,576 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T13:18:27,576 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1305942291, stopped=false 2024-11-11T13:18:27,577 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=37d26ecf6460,37611,1731331103338 2024-11-11T13:18:27,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:27,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:27,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T13:18:27,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:27,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:27,618 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T13:18:27,618 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:27,618 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T13:18:27,618 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:27,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:27,619 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37d26ecf6460,44079,1731331103552' ***** 2024-11-11T13:18:27,619 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T13:18:27,619 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '37d26ecf6460,33597,1731331106961' ***** 2024-11-11T13:18:27,619 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T13:18:27,619 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:27,619 INFO [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T13:18:27,619 INFO [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence 2024-11-11T13:18:27,619 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T13:18:27,619 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T13:18:27,619 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T13:18:27,619 INFO [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1739): Disabling cache 2024-11-11T13:18:27,619 INFO [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T13:18:27,619 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-11T13:18:27,619 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-11T13:18:27,619 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:27,619 INFO [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/9af2702f-1547-b68b-0b9e-3a05bbf495f2/bucket.persistence 2024-11-11T13:18:27,619 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T13:18:27,620 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-11T13:18:27,620 INFO [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-11T13:18:27,620 DEBUG [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1488): Thread RS:1;37d26ecf6460:44079 started persisting bucket cache to file 2024-11-11T13:18:27,620 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-11T13:18:27,620 DEBUG [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:27,620 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T13:18:27,620 DEBUG [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:27,620 DEBUG [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1488): Thread RS:2;37d26ecf6460:33597 started persisting bucket cache to file 2024-11-11T13:18:27,620 DEBUG [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-11T13:18:27,620 DEBUG [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-11T13:18:27,623 DEBUG [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-11T13:18:27,623 DEBUG [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731331107620 2024-11-11T13:18:27,623 DEBUG [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-11T13:18:27,623 DEBUG [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731331107620 2024-11-11T13:18:27,623 DEBUG [RS:1;37d26ecf6460:44079 {}] bucket.BucketCache(1510): Thread RS:1;37d26ecf6460:44079 finished persisting bucket cache to file, renaming 2024-11-11T13:18:27,623 DEBUG [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1510): Thread RS:2;37d26ecf6460:33597 finished persisting bucket cache to file, renaming 2024-11-11T13:18:27,623 WARN [RS:2;37d26ecf6460:33597 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-11T13:18:27,623 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T13:18:27,623 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T13:18:27,623 INFO [RS:1;37d26ecf6460:44079 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T13:18:27,623 INFO [RS:2;37d26ecf6460:33597 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T13:18:27,623 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T13:18:27,623 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T13:18:27,623 INFO [RS:1;37d26ecf6460:44079 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T13:18:27,623 INFO [RS:2;37d26ecf6460:33597 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T13:18:27,623 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(959): stopping server 37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,623 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(3091): Received CLOSE for ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,623 INFO [RS:2;37d26ecf6460:33597 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:18:27,623 INFO [RS:2;37d26ecf6460:33597 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;37d26ecf6460:33597. 2024-11-11T13:18:27,623 DEBUG [RS:2;37d26ecf6460:33597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:27,623 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(959): stopping server 37d26ecf6460,44079,1731331103552 2024-11-11T13:18:27,624 DEBUG [RS:2;37d26ecf6460:33597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:27,624 INFO [RS:1;37d26ecf6460:44079 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:18:27,624 INFO [RS:1;37d26ecf6460:44079 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;37d26ecf6460:44079. 2024-11-11T13:18:27,624 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(976): stopping server 37d26ecf6460,33597,1731331106961; all regions closed. 2024-11-11T13:18:27,624 DEBUG [RS:1;37d26ecf6460:44079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T13:18:27,624 DEBUG [RS:1;37d26ecf6460:44079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:27,624 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T13:18:27,624 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T13:18:27,624 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ddb4e4f7d13ad045e0af24a7d77999d6, disabling compactions & flushes 2024-11-11T13:18:27,624 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T13:18:27,624 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,624 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T13:18:27,624 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,624 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T13:18:27,624 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. after waiting 0 ms 2024-11-11T13:18:27,624 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,624 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-11T13:18:27,624 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1325): Online Regions={ddb4e4f7d13ad045e0af24a7d77999d6=testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6., 1588230740=hbase:meta,,1.1588230740} 2024-11-11T13:18:27,624 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-11T13:18:27,624 DEBUG [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, ddb4e4f7d13ad045e0af24a7d77999d6 2024-11-11T13:18:27,625 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T13:18:27,625 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T13:18:27,625 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T13:18:27,625 DEBUG [StoreFileCloser-ddb4e4f7d13ad045e0af24a7d77999d6-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/family/98289c58989c4690b089a1ca3eb22d71 2024-11-11T13:18:27,625 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T13:18:27,625 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T13:18:27,625 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-11T13:18:27,625 DEBUG [RS:2;37d26ecf6460:33597 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs 2024-11-11T13:18:27,625 INFO [RS:2;37d26ecf6460:33597 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C33597%2C1731331106961:(num 1731331107185) 2024-11-11T13:18:27,626 DEBUG [RS:2;37d26ecf6460:33597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] hbase.ChoreService(370): Chore service for: regionserver/37d26ecf6460:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T13:18:27,626 INFO [regionserver/37d26ecf6460:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:18:27,626 INFO [RS:2;37d26ecf6460:33597 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33597 2024-11-11T13:18:27,630 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/default/testBlockEvictionOnGracefulStop1731331104738/ddb4e4f7d13ad045e0af24a7d77999d6/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-11T13:18:27,630 INFO [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,630 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ddb4e4f7d13ad045e0af24a7d77999d6: Waiting for close lock at 1731331107624Running coprocessor pre-close hooks at 1731331107624Disabling compacts and flushes for region at 1731331107624Disabling writes for close at 1731331107624Writing region close event to WAL at 1731331107625 (+1 ms)Running coprocessor post-close hooks at 1731331107630 (+5 ms)Closed at 1731331107630 2024-11-11T13:18:27,631 DEBUG [RS_CLOSE_REGION-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6. 2024-11-11T13:18:27,634 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b2b90383832041f2b88183d7256eb2df 2024-11-11T13:18:27,641 INFO [regionserver/37d26ecf6460:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:27,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37d26ecf6460,33597,1731331106961 2024-11-11T13:18:27,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:18:27,645 INFO [RS:2;37d26ecf6460:33597 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:18:27,647 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/info/b2b90383832041f2b88183d7256eb2df is 209, key is testBlockEvictionOnGracefulStop1731331104738,,1731331104756.ddb4e4f7d13ad045e0af24a7d77999d6./info:regioninfo/1731331107071/Put/seqid=0 2024-11-11T13:18:27,648 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/info/b2b90383832041f2b88183d7256eb2df 2024-11-11T13:18:27,653 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37d26ecf6460,33597,1731331106961] 2024-11-11T13:18:27,656 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ee63b9bbfcc4519bc38ae639d3eced0 2024-11-11T13:18:27,661 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37d26ecf6460,33597,1731331106961 already deleted, retry=false 2024-11-11T13:18:27,661 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 37d26ecf6460,33597,1731331106961 expired; onlineServers=1 2024-11-11T13:18:27,668 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/ns/8ee63b9bbfcc4519bc38ae639d3eced0 is 43, key is default/ns:d/1731331104641/Put/seqid=0 2024-11-11T13:18:27,669 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/ns/8ee63b9bbfcc4519bc38ae639d3eced0 2024-11-11T13:18:27,676 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4885a7f6f91747a79fc3e891848be0cd 2024-11-11T13:18:27,689 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/table/4885a7f6f91747a79fc3e891848be0cd is 80, key is testBlockEvictionOnGracefulStop1731331104738/table:state/1731331105180/Put/seqid=0 2024-11-11T13:18:27,690 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/table/4885a7f6f91747a79fc3e891848be0cd 2024-11-11T13:18:27,692 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/info/b2b90383832041f2b88183d7256eb2df as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/b2b90383832041f2b88183d7256eb2df 2024-11-11T13:18:27,693 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/b2b90383832041f2b88183d7256eb2df, delay=962 ms 2024-11-11T13:18:27,694 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/b2b90383832041f2b88183d7256eb2df, entries=14, sequenceid=13, filesize=7.5 K 2024-11-11T13:18:27,694 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/ns/8ee63b9bbfcc4519bc38ae639d3eced0 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/8ee63b9bbfcc4519bc38ae639d3eced0 2024-11-11T13:18:27,696 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/8ee63b9bbfcc4519bc38ae639d3eced0, delay=961 ms 2024-11-11T13:18:27,697 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/8ee63b9bbfcc4519bc38ae639d3eced0, entries=2, sequenceid=13, filesize=5.0 K 2024-11-11T13:18:27,697 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/.tmp/table/4885a7f6f91747a79fc3e891848be0cd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4885a7f6f91747a79fc3e891848be0cd 2024-11-11T13:18:27,698 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4885a7f6f91747a79fc3e891848be0cd, delay=914 ms 2024-11-11T13:18:27,699 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4885a7f6f91747a79fc3e891848be0cd, entries=2, sequenceid=13, filesize=5.3 K 2024-11-11T13:18:27,700 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=13, compaction requested=false 2024-11-11T13:18:27,700 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/info/b2b90383832041f2b88183d7256eb2df 2024-11-11T13:18:27,701 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/ns/8ee63b9bbfcc4519bc38ae639d3eced0 2024-11-11T13:18:27,702 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/table/4885a7f6f91747a79fc3e891848be0cd 2024-11-11T13:18:27,706 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-11T13:18:27,706 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T13:18:27,706 INFO [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T13:18:27,706 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731331107624Running coprocessor pre-close hooks at 1731331107624Disabling compacts and flushes for region at 1731331107624Disabling writes for close at 1731331107625 (+1 ms)Obtaining lock to block concurrent updates at 1731331107625Preparing flush snapshotting stores in 1588230740 at 1731331107625Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731331107626 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731331107626Flushing 1588230740/info: creating writer at 1731331107626Flushing 1588230740/info: appending metadata at 1731331107647 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731331107647Flushing 1588230740/ns: creating writer at 1731331107650 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731331107667 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1731331107668 (+1 ms)Flushing 1588230740/table: creating writer at 1731331107671 (+3 ms)Flushing 1588230740/table: appending metadata at 1731331107688 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731331107688Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38713d09: reopening flushed file at 1731331107692 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34c8d435: reopening flushed file at 1731331107694 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ab2727: reopening flushed file at 1731331107697 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=13, compaction requested=false at 1731331107700 (+3 ms)Writing region close event to WAL at 1731331107702 (+2 ms)Running coprocessor post-close hooks at 1731331107706 (+4 ms)Closed at 1731331107706 2024-11-11T13:18:27,707 DEBUG [RS_CLOSE_META-regionserver/37d26ecf6460:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T13:18:27,710 INFO [regionserver/37d26ecf6460:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:27,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:27,753 INFO [RS:2;37d26ecf6460:33597 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:18:27,753 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33597-0x1012b55cc430003, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:27,753 INFO [RS:2;37d26ecf6460:33597 {}] regionserver.HRegionServer(1031): Exiting; stopping=37d26ecf6460,33597,1731331106961; zookeeper connection closed. 2024-11-11T13:18:27,753 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4bacf699 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4bacf699 2024-11-11T13:18:27,825 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(976): stopping server 37d26ecf6460,44079,1731331103552; all regions closed. 2024-11-11T13:18:27,831 DEBUG [RS:1;37d26ecf6460:44079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs 2024-11-11T13:18:27,831 INFO [RS:1;37d26ecf6460:44079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C44079%2C1731331103552.meta:.meta(num 1731331104582) 2024-11-11T13:18:27,836 DEBUG [RS:1;37d26ecf6460:44079 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/oldWALs 2024-11-11T13:18:27,836 INFO [RS:1;37d26ecf6460:44079 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 37d26ecf6460%2C44079%2C1731331103552:(num 1731331104256) 2024-11-11T13:18:27,836 DEBUG [RS:1;37d26ecf6460:44079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T13:18:27,836 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T13:18:27,836 INFO [RS:1;37d26ecf6460:44079 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:18:27,836 INFO [RS:1;37d26ecf6460:44079 {}] hbase.ChoreService(370): Chore service for: regionserver/37d26ecf6460:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T13:18:27,837 INFO [RS:1;37d26ecf6460:44079 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:18:27,837 INFO [regionserver/37d26ecf6460:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:18:27,837 INFO [RS:1;37d26ecf6460:44079 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44079 2024-11-11T13:18:27,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/37d26ecf6460,44079,1731331103552 2024-11-11T13:18:27,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T13:18:27,845 INFO [RS:1;37d26ecf6460:44079 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:18:27,846 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [37d26ecf6460,44079,1731331103552] 2024-11-11T13:18:27,869 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/37d26ecf6460,44079,1731331103552 already deleted, retry=false 2024-11-11T13:18:27,869 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 37d26ecf6460,44079,1731331103552 expired; onlineServers=0 2024-11-11T13:18:27,869 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '37d26ecf6460,37611,1731331103338' ***** 2024-11-11T13:18:27,869 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T13:18:27,870 INFO [M:0;37d26ecf6460:37611 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T13:18:27,870 INFO [M:0;37d26ecf6460:37611 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T13:18:27,870 DEBUG [M:0;37d26ecf6460:37611 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T13:18:27,870 DEBUG [M:0;37d26ecf6460:37611 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T13:18:27,870 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T13:18:27,870 DEBUG [master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731331103946 {}] cleaner.HFileCleaner(306): Exit Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.large.0-1731331103946,5,FailOnTimeoutGroup] 2024-11-11T13:18:27,870 DEBUG [master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731331103946 {}] cleaner.HFileCleaner(306): Exit Thread[master/37d26ecf6460:0:becomeActiveMaster-HFileCleaner.small.0-1731331103946,5,FailOnTimeoutGroup] 2024-11-11T13:18:27,871 INFO [M:0;37d26ecf6460:37611 {}] hbase.ChoreService(370): Chore service for: master/37d26ecf6460:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T13:18:27,871 INFO [M:0;37d26ecf6460:37611 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T13:18:27,871 DEBUG [M:0;37d26ecf6460:37611 {}] master.HMaster(1795): Stopping service threads 2024-11-11T13:18:27,871 INFO [M:0;37d26ecf6460:37611 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T13:18:27,871 INFO [M:0;37d26ecf6460:37611 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T13:18:27,872 INFO [M:0;37d26ecf6460:37611 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T13:18:27,872 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T13:18:27,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T13:18:27,878 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T13:18:27,878 DEBUG [M:0;37d26ecf6460:37611 {}] zookeeper.ZKUtil(347): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T13:18:27,878 WARN [M:0;37d26ecf6460:37611 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T13:18:27,879 INFO [M:0;37d26ecf6460:37611 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/.lastflushedseqids 2024-11-11T13:18:27,888 INFO [M:0;37d26ecf6460:37611 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T13:18:27,888 INFO [M:0;37d26ecf6460:37611 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T13:18:27,889 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T13:18:27,889 INFO [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:27,889 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:27,889 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T13:18:27,889 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:27,889 INFO [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.84 KB heapSize=41.09 KB 2024-11-11T13:18:27,900 DEBUG [M:0;37d26ecf6460:37611 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 09e27ceb3bdd4f24ba843d76f6c48029 2024-11-11T13:18:27,913 DEBUG [M:0;37d26ecf6460:37611 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09e27ceb3bdd4f24ba843d76f6c48029 is 82, key is hbase:meta,,1/info:regioninfo/1731331104620/Put/seqid=0 2024-11-11T13:18:27,913 INFO [M:0;37d26ecf6460:37611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09e27ceb3bdd4f24ba843d76f6c48029 2024-11-11T13:18:27,921 DEBUG [M:0;37d26ecf6460:37611 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e8e516a99a724bd78151ea3f4e8d3ea5 2024-11-11T13:18:27,935 DEBUG [M:0;37d26ecf6460:37611 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8e516a99a724bd78151ea3f4e8d3ea5 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731331105186/Put/seqid=0 2024-11-11T13:18:27,936 INFO [M:0;37d26ecf6460:37611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.10 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8e516a99a724bd78151ea3f4e8d3ea5 2024-11-11T13:18:27,943 DEBUG [M:0;37d26ecf6460:37611 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2f861a8e872e45998de44b392733ecc9 2024-11-11T13:18:27,956 DEBUG [M:0;37d26ecf6460:37611 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2f861a8e872e45998de44b392733ecc9 is 69, key is 37d26ecf6460,33597,1731331106961/rs:state/1731331107022/Put/seqid=0 2024-11-11T13:18:27,957 INFO [M:0;37d26ecf6460:37611 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2f861a8e872e45998de44b392733ecc9 2024-11-11T13:18:27,959 INFO [M:0;37d26ecf6460:37611 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2f861a8e872e45998de44b392733ecc9 2024-11-11T13:18:27,959 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09e27ceb3bdd4f24ba843d76f6c48029 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09e27ceb3bdd4f24ba843d76f6c48029 2024-11-11T13:18:27,961 INFO [M:0;37d26ecf6460:37611 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09e27ceb3bdd4f24ba843d76f6c48029, entries=8, sequenceid=83, filesize=5.5 K 2024-11-11T13:18:27,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:27,961 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/e8e516a99a724bd78151ea3f4e8d3ea5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8e516a99a724bd78151ea3f4e8d3ea5 2024-11-11T13:18:27,961 INFO [RS:1;37d26ecf6460:44079 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:18:27,961 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44079-0x1012b55cc430002, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:27,961 INFO [RS:1;37d26ecf6460:44079 {}] regionserver.HRegionServer(1031): Exiting; stopping=37d26ecf6460,44079,1731331103552; zookeeper connection closed. 2024-11-11T13:18:27,962 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6f262de9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6f262de9 2024-11-11T13:18:27,962 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-11T13:18:27,963 INFO [M:0;37d26ecf6460:37611 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/e8e516a99a724bd78151ea3f4e8d3ea5, entries=9, sequenceid=83, filesize=6.4 K 2024-11-11T13:18:27,963 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2f861a8e872e45998de44b392733ecc9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2f861a8e872e45998de44b392733ecc9 2024-11-11T13:18:27,965 INFO [M:0;37d26ecf6460:37611 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 2f861a8e872e45998de44b392733ecc9 2024-11-11T13:18:27,965 INFO [M:0;37d26ecf6460:37611 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/be1f0dc0-ab75-316e-db8e-b5c8659b2142/5a3eaf9d-d74f-83b0-4cf2-b0be150fc4c2/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2f861a8e872e45998de44b392733ecc9, entries=3, sequenceid=83, filesize=5.3 K 2024-11-11T13:18:27,966 INFO [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=83, compaction requested=false 2024-11-11T13:18:27,967 INFO [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T13:18:27,968 DEBUG [M:0;37d26ecf6460:37611 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731331107889Disabling compacts and flushes for region at 1731331107889Disabling writes for close at 1731331107889Obtaining lock to block concurrent updates at 1731331107889Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731331107889Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33626, getHeapSize=42008, getOffHeapSize=0, getCellsCount=98 at 1731331107890 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731331107890Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731331107890Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731331107912 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731331107912Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731331107915 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731331107935 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731331107935Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731331107938 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731331107955 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731331107956 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@150b93d2: reopening flushed file at 1731331107959 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f5e654f: reopening flushed file at 1731331107961 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f4bec94: reopening flushed file at 1731331107963 (+2 ms)Finished flush of dataSize ~32.84 KB/33626, heapSize ~40.79 KB/41768, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 77ms, sequenceid=83, compaction requested=false at 1731331107966 (+3 ms)Writing region close event to WAL at 1731331107967 (+1 ms)Closed at 1731331107967 2024-11-11T13:18:27,969 INFO [M:0;37d26ecf6460:37611 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T13:18:27,969 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T13:18:27,969 INFO [M:0;37d26ecf6460:37611 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37611 2024-11-11T13:18:27,969 INFO [M:0;37d26ecf6460:37611 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T13:18:28,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:28,078 INFO [M:0;37d26ecf6460:37611 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T13:18:28,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37611-0x1012b55cc430000, quorum=127.0.0.1:49804, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T13:18:28,081 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T13:18:28,082 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-11T13:18:28,082 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T13:18:28,087 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=239 (was 260), ProcessCount=11 (was 11), AvailableMemoryMB=437 (was 549)