2024-11-25 11:00:14,480 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 11:00:14,529 main DEBUG Took 0.046131 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-25 11:00:14,529 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-25 11:00:14,530 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-25 11:00:14,531 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-25 11:00:14,533 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,548 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-25 11:00:14,576 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,578 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,581 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,581 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,586 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,586 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,590 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,591 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,592 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,592 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,593 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,594 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,594 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,595 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,596 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,596 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,597 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,597 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,598 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,598 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,599 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,599 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,600 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,600 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-25 11:00:14,600 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,601 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-25 11:00:14,603 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-25 11:00:14,605 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-25 11:00:14,607 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-25 11:00:14,608 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-25 11:00:14,609 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-25 11:00:14,610 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-25 11:00:14,621 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-25 11:00:14,624 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-25 11:00:14,626 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-25 11:00:14,627 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-25 11:00:14,627 main DEBUG createAppenders(={Console}) 2024-11-25 11:00:14,628 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-25 11:00:14,628 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-25 11:00:14,629 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-25 11:00:14,629 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-25 11:00:14,630 main DEBUG OutputStream closed 2024-11-25 11:00:14,630 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-25 11:00:14,630 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-25 11:00:14,631 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-25 11:00:14,734 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-25 11:00:14,736 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-25 11:00:14,737 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-25 11:00:14,739 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-25 11:00:14,740 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-25 11:00:14,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-25 11:00:14,741 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-25 11:00:14,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-25 11:00:14,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-25 11:00:14,742 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-25 11:00:14,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-25 11:00:14,743 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-25 11:00:14,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-25 11:00:14,744 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-25 11:00:14,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-25 11:00:14,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-25 11:00:14,745 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-25 11:00:14,746 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-25 11:00:14,749 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25 11:00:14,750 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-25 11:00:14,750 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-25 11:00:14,751 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-25T11:00:14,766 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-25 11:00:14,769 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-25 11:00:14,769 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-25T11:00:15,192 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1 2024-11-25T11:00:15,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=907, ProcessCount=11, AvailableMemoryMB=4449 2024-11-25T11:00:15,396 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-25T11:00:15,558 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:15,585 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/cluster_76d2f9c3-471d-e1f1-50ed-4fdc86125d41, deleteOnExit=true 2024-11-25T11:00:15,738 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/cluster_76d2f9c3-471d-e1f1-50ed-4fdc86125d41/zookeeper_0, clientPort=55243, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/cluster_76d2f9c3-471d-e1f1-50ed-4fdc86125d41/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/cluster_76d2f9c3-471d-e1f1-50ed-4fdc86125d41/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T11:00:15,754 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55243 2024-11-25T11:00:15,754 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:15,887 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d with version=8 2024-11-25T11:00:15,888 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/hbase-staging 2024-11-25T11:00:16,021 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-25T11:00:16,344 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T11:00:16,357 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:16,358 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:16,366 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T11:00:16,366 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:16,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T11:00:16,551 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T11:00:16,674 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T11:00:16,727 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-25T11:00:16,740 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-25T11:00:16,745 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T11:00:16,794 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 14639 (auto-detected) 2024-11-25T11:00:16,795 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-25T11:00:16,826 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43465 2024-11-25T11:00:16,861 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:43465 connecting to ZooKeeper ensemble=127.0.0.1:55243 2024-11-25T11:00:16,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:434650x0, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T11:00:16,910 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:43465-0x1007698bf5e0000 connected 2024-11-25T11:00:16,942 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:16,944 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:16,954 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:16,959 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d, hbase.cluster.distributed=false 2024-11-25T11:00:16,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T11:00:16,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43465 2024-11-25T11:00:16,997 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43465 2024-11-25T11:00:16,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43465 2024-11-25T11:00:17,000 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43465 2024-11-25T11:00:17,003 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43465 2024-11-25T11:00:17,111 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T11:00:17,113 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:17,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:17,114 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T11:00:17,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:17,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T11:00:17,119 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T11:00:17,123 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T11:00:17,124 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36179 2024-11-25T11:00:17,127 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36179 connecting to ZooKeeper ensemble=127.0.0.1:55243 2024-11-25T11:00:17,127 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:17,129 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:17,133 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361790x0, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T11:00:17,134 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36179-0x1007698bf5e0001 connected 2024-11-25T11:00:17,135 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:17,143 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T11:00:17,160 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.cache 2024-11-25T11:00:17,168 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T11:00:17,170 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T11:00:17,172 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T11:00:17,175 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T11:00:17,175 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T11:00:17,183 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T11:00:17,188 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T11:00:17,188 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.persistence is complete. 2024-11-25T11:00:17,194 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T11:00:17,199 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T11:00:17,211 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T11:00:17,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36179 2024-11-25T11:00:17,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36179 2024-11-25T11:00:17,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36179 2024-11-25T11:00:17,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36179 2024-11-25T11:00:17,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36179 2024-11-25T11:00:17,277 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T11:00:17,277 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:17,279 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:17,280 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T11:00:17,280 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:17,280 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T11:00:17,282 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T11:00:17,289 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T11:00:17,312 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35919 2024-11-25T11:00:17,315 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35919 connecting to ZooKeeper ensemble=127.0.0.1:55243 2024-11-25T11:00:17,316 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:17,321 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:17,338 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359190x0, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T11:00:17,339 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359190x0, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:17,340 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T11:00:17,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35919-0x1007698bf5e0002 connected 2024-11-25T11:00:17,345 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.cache 2024-11-25T11:00:17,346 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T11:00:17,346 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T11:00:17,347 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T11:00:17,348 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T11:00:17,348 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T11:00:17,350 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T11:00:17,350 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.persistence is complete. 2024-11-25T11:00:17,352 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T11:00:17,354 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T11:00:17,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T11:00:17,359 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T11:00:17,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35919 2024-11-25T11:00:17,389 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35919 2024-11-25T11:00:17,391 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35919 2024-11-25T11:00:17,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35919 2024-11-25T11:00:17,397 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35919 2024-11-25T11:00:17,423 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6bfc37bad115:43465 2024-11-25T11:00:17,424 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6bfc37bad115,43465,1732532416119 2024-11-25T11:00:17,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:17,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:17,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:17,438 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6bfc37bad115,43465,1732532416119 2024-11-25T11:00:17,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T11:00:17,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T11:00:17,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:17,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:17,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:17,488 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T11:00:17,492 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6bfc37bad115,43465,1732532416119 from backup master directory 2024-11-25T11:00:17,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:17,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6bfc37bad115,43465,1732532416119 2024-11-25T11:00:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:17,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:17,500 WARN [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T11:00:17,500 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6bfc37bad115,43465,1732532416119 2024-11-25T11:00:17,503 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-25T11:00:17,507 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-25T11:00:17,526 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/hbase.id] with ID: b31e9512-4557-46bb-8739-7bf24bbe80df 2024-11-25T11:00:17,526 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.tmp/hbase.id 2024-11-25T11:00:17,535 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/hbase.id] 2024-11-25T11:00:17,624 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:17,631 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T11:00:17,666 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 32ms. 2024-11-25T11:00:17,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:17,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:17,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:17,717 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T11:00:17,721 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T11:00:17,789 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T11:00:17,797 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:17,905 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store 2024-11-25T11:00:17,960 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-25T11:00:17,967 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:17,969 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T11:00:17,969 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:17,970 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:17,972 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T11:00:17,972 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:17,972 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:17,973 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732532417969Disabling compacts and flushes for region at 1732532417969Disabling writes for close at 1732532417972 (+3 ms)Writing region close event to WAL at 1732532417972Closed at 1732532417972 2024-11-25T11:00:17,974 WARN [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/.initializing 2024-11-25T11:00:17,974 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/WALs/6bfc37bad115,43465,1732532416119 2024-11-25T11:00:18,005 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T11:00:18,024 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6bfc37bad115%2C43465%2C1732532416119, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/WALs/6bfc37bad115,43465,1732532416119, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/oldWALs, maxLogs=10 2024-11-25T11:00:18,076 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/WALs/6bfc37bad115,43465,1732532416119/6bfc37bad115%2C43465%2C1732532416119.1732532418031 2024-11-25T11:00:18,077 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:18,078 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:18,079 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:18,084 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,085 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,144 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,186 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T11:00:18,192 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,194 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:18,195 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,202 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T11:00:18,203 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,203 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:18,204 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T11:00:18,215 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,215 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:18,216 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T11:00:18,227 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:18,228 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,233 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,234 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,238 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,239 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,243 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T11:00:18,246 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:18,258 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T11:00:18,259 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65365351, jitterRate=-0.02598036825656891}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T11:00:18,270 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732532418103Initializing all the Stores at 1732532418104 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532418104Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532418108 (+4 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532418109 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532418109Cleaning up temporary data from old regions at 1732532418239 (+130 ms)Region opened successfully at 1732532418269 (+30 ms) 2024-11-25T11:00:18,272 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T11:00:18,334 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c9a7dc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T11:00:18,390 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T11:00:18,408 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T11:00:18,408 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T11:00:18,413 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T11:00:18,414 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T11:00:18,430 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-25T11:00:18,431 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T11:00:18,473 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T11:00:18,505 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T11:00:18,508 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T11:00:18,515 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T11:00:18,518 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T11:00:18,522 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T11:00:18,525 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T11:00:18,530 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T11:00:18,538 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T11:00:18,540 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T11:00:18,542 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T11:00:18,567 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T11:00:18,569 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T11:00:18,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:18,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:18,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:18,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,596 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6bfc37bad115,43465,1732532416119, sessionid=0x1007698bf5e0000, setting cluster-up flag (Was=false) 2024-11-25T11:00:18,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,612 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,621 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T11:00:18,624 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6bfc37bad115,43465,1732532416119 2024-11-25T11:00:18,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:18,636 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T11:00:18,639 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6bfc37bad115,43465,1732532416119 2024-11-25T11:00:18,645 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T11:00:18,717 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(746): ClusterId : b31e9512-4557-46bb-8739-7bf24bbe80df 2024-11-25T11:00:18,717 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(746): ClusterId : b31e9512-4557-46bb-8739-7bf24bbe80df 2024-11-25T11:00:18,720 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T11:00:18,720 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T11:00:18,727 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T11:00:18,727 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T11:00:18,728 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T11:00:18,728 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T11:00:18,732 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T11:00:18,733 DEBUG [RS:1;6bfc37bad115:35919 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4543145d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T11:00:18,736 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:18,738 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T11:00:18,739 DEBUG [RS:0;6bfc37bad115:36179 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13b6658b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T11:00:18,753 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6bfc37bad115:36179 2024-11-25T11:00:18,754 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T11:00:18,755 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6bfc37bad115:35919 2024-11-25T11:00:18,757 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T11:00:18,757 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T11:00:18,757 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T11:00:18,757 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T11:00:18,757 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T11:00:18,757 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T11:00:18,760 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(2659): reportForDuty to master=6bfc37bad115,43465,1732532416119 with port=36179, startcode=1732532417058 2024-11-25T11:00:18,760 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(2659): reportForDuty to master=6bfc37bad115,43465,1732532416119 with port=35919, startcode=1732532417275 2024-11-25T11:00:18,766 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T11:00:18,775 DEBUG [RS:0;6bfc37bad115:36179 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T11:00:18,775 DEBUG [RS:1;6bfc37bad115:35919 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T11:00:18,774 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6bfc37bad115,43465,1732532416119 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T11:00:18,784 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:18,784 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:18,785 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:18,785 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:18,785 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6bfc37bad115:0, corePoolSize=10, maxPoolSize=10 2024-11-25T11:00:18,785 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:18,786 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T11:00:18,786 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:18,787 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732532448787 2024-11-25T11:00:18,790 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T11:00:18,791 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T11:00:18,793 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:18,794 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T11:00:18,795 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T11:00:18,796 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T11:00:18,796 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T11:00:18,796 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T11:00:18,810 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,810 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T11:00:18,813 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:18,819 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T11:00:18,820 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T11:00:18,821 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T11:00:18,831 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T11:00:18,831 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T11:00:18,841 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532418838,5,FailOnTimeoutGroup] 2024-11-25T11:00:18,842 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532418841,5,FailOnTimeoutGroup] 2024-11-25T11:00:18,842 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:18,842 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T11:00:18,843 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:18,844 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:18,851 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T11:00:18,851 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:18,879 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:18,886 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T11:00:18,888 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55465, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T11:00:18,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35175, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T11:00:18,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T11:00:18,898 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43465 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6bfc37bad115,35919,1732532417275 2024-11-25T11:00:18,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:18,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T11:00:18,907 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43465 {}] master.ServerManager(517): Registering regionserver=6bfc37bad115,35919,1732532417275 2024-11-25T11:00:18,922 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T11:00:18,922 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,922 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:18,923 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T11:00:18,925 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43465 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6bfc37bad115,36179,1732532417058 2024-11-25T11:00:18,925 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=43465 {}] master.ServerManager(517): Registering regionserver=6bfc37bad115,36179,1732532417058 2024-11-25T11:00:18,927 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:18,927 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T11:00:18,927 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T11:00:18,931 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:18,931 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T11:00:18,931 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T11:00:18,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:18,939 DEBUG [RS:1;6bfc37bad115:35919 {}] zookeeper.ZKUtil(111): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6bfc37bad115,35919,1732532417275 2024-11-25T11:00:18,940 WARN [RS:1;6bfc37bad115:35919 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T11:00:18,940 INFO [RS:1;6bfc37bad115:35919 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:18,940 DEBUG [RS:0;6bfc37bad115:36179 {}] zookeeper.ZKUtil(111): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6bfc37bad115,36179,1732532417058 2024-11-25T11:00:18,940 WARN [RS:0;6bfc37bad115:36179 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T11:00:18,940 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,35919,1732532417275 2024-11-25T11:00:18,941 INFO [RS:0;6bfc37bad115:36179 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:18,941 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36179,1732532417058 2024-11-25T11:00:18,947 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6bfc37bad115,35919,1732532417275] 2024-11-25T11:00:18,947 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6bfc37bad115,36179,1732532417058] 2024-11-25T11:00:18,950 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T11:00:18,950 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:18,951 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T11:00:18,984 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T11:00:18,985 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:18,985 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:18,986 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T11:00:18,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:18,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:18,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T11:00:18,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T11:00:18,988 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T11:00:18,989 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T11:00:19,002 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T11:00:19,005 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T11:00:19,044 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T11:00:19,045 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68825847, jitterRate=0.02558504045009613}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T11:00:19,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732532418879Initializing all the Stores at 1732532418880 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532418880Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532418880Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532418880Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532418880Cleaning up temporary data from old regions at 1732532418988 (+108 ms)Region opened successfully at 1732532419047 (+59 ms) 2024-11-25T11:00:19,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T11:00:19,048 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T11:00:19,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T11:00:19,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T11:00:19,048 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T11:00:19,051 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T11:00:19,051 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T11:00:19,057 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:19,058 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732532419048Disabling compacts and flushes for region at 1732532419048Disabling writes for close at 1732532419048Writing region close event to WAL at 1732532419057 (+9 ms)Closed at 1732532419057 2024-11-25T11:00:19,061 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:19,061 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T11:00:19,063 INFO [RS:1;6bfc37bad115:35919 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T11:00:19,063 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,064 INFO [RS:0;6bfc37bad115:36179 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T11:00:19,064 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,065 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T11:00:19,068 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T11:00:19,072 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T11:00:19,074 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,074 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,074 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,074 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T11:00:19,074 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,075 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,075 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,075 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,075 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,075 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,075 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T11:00:19,075 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,075 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,075 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T11:00:19,076 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,076 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,077 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:19,077 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,077 DEBUG [RS:0;6bfc37bad115:36179 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:19,077 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,077 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:19,077 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:19,078 DEBUG [RS:1;6bfc37bad115:35919 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:19,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T11:00:19,091 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T11:00:19,094 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T11:00:19,097 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,097 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,097 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,097 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,097 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,097 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,097 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,097 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,098 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,098 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,36179,1732532417058-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T11:00:19,098 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,106 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,35919,1732532417275-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T11:00:19,140 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T11:00:19,140 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T11:00:19,143 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,36179,1732532417058-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,143 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,35919,1732532417275-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,143 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,143 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.Replication(171): 6bfc37bad115,36179,1732532417058 started 2024-11-25T11:00:19,143 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,145 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.Replication(171): 6bfc37bad115,35919,1732532417275 started 2024-11-25T11:00:19,174 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,174 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:19,177 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1482): Serving as 6bfc37bad115,35919,1732532417275, RpcServer on 6bfc37bad115/172.17.0.3:35919, sessionid=0x1007698bf5e0002 2024-11-25T11:00:19,177 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1482): Serving as 6bfc37bad115,36179,1732532417058, RpcServer on 6bfc37bad115/172.17.0.3:36179, sessionid=0x1007698bf5e0001 2024-11-25T11:00:19,178 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T11:00:19,178 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T11:00:19,178 DEBUG [RS:0;6bfc37bad115:36179 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6bfc37bad115,36179,1732532417058 2024-11-25T11:00:19,178 DEBUG [RS:1;6bfc37bad115:35919 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6bfc37bad115,35919,1732532417275 2024-11-25T11:00:19,179 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,36179,1732532417058' 2024-11-25T11:00:19,179 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,35919,1732532417275' 2024-11-25T11:00:19,179 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T11:00:19,179 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T11:00:19,180 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T11:00:19,180 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T11:00:19,181 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T11:00:19,181 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T11:00:19,181 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T11:00:19,181 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T11:00:19,181 DEBUG [RS:0;6bfc37bad115:36179 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6bfc37bad115,36179,1732532417058 2024-11-25T11:00:19,182 DEBUG [RS:1;6bfc37bad115:35919 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6bfc37bad115,35919,1732532417275 2024-11-25T11:00:19,182 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,36179,1732532417058' 2024-11-25T11:00:19,182 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,35919,1732532417275' 2024-11-25T11:00:19,182 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T11:00:19,182 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T11:00:19,182 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T11:00:19,182 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T11:00:19,183 DEBUG [RS:0;6bfc37bad115:36179 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T11:00:19,184 DEBUG [RS:1;6bfc37bad115:35919 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T11:00:19,184 INFO [RS:0;6bfc37bad115:36179 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T11:00:19,184 INFO [RS:1;6bfc37bad115:35919 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T11:00:19,184 INFO [RS:0;6bfc37bad115:36179 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T11:00:19,184 INFO [RS:1;6bfc37bad115:35919 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T11:00:19,247 WARN [6bfc37bad115:43465 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T11:00:19,296 INFO [RS:1;6bfc37bad115:35919 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T11:00:19,297 INFO [RS:1;6bfc37bad115:35919 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C35919%2C1732532417275, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,35919,1732532417275, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:19,301 INFO [RS:0;6bfc37bad115:36179 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T11:00:19,302 INFO [RS:0;6bfc37bad115:36179 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C36179%2C1732532417058, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36179,1732532417058, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:19,333 INFO [RS:0;6bfc37bad115:36179 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36179,1732532417058/6bfc37bad115%2C36179%2C1732532417058.1732532419304 2024-11-25T11:00:19,334 INFO [RS:1;6bfc37bad115:35919 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,35919,1732532417275/6bfc37bad115%2C35919%2C1732532417275.1732532419301 2024-11-25T11:00:19,334 DEBUG [RS:0;6bfc37bad115:36179 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:19,337 DEBUG [RS:1;6bfc37bad115:35919 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:19,501 DEBUG [6bfc37bad115:43465 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T11:00:19,513 DEBUG [6bfc37bad115:43465 {}] balancer.BalancerClusterState(204): Hosts are {6bfc37bad115=0} racks are {/default-rack=0} 2024-11-25T11:00:19,516 DEBUG [6bfc37bad115:43465 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T11:00:19,517 DEBUG [6bfc37bad115:43465 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T11:00:19,517 DEBUG [6bfc37bad115:43465 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T11:00:19,517 DEBUG [6bfc37bad115:43465 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T11:00:19,517 INFO [6bfc37bad115:43465 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T11:00:19,517 INFO [6bfc37bad115:43465 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T11:00:19,517 DEBUG [6bfc37bad115:43465 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T11:00:19,528 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6bfc37bad115,36179,1732532417058 2024-11-25T11:00:19,536 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,36179,1732532417058, state=OPENING 2024-11-25T11:00:19,543 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T11:00:19,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:19,545 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:19,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:19,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:19,547 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:19,548 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:19,550 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T11:00:19,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,36179,1732532417058}] 2024-11-25T11:00:19,736 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T11:00:19,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55087, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T11:00:19,753 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T11:00:19,754 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:19,755 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T11:00:19,756 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C36179%2C1732532417058.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36179,1732532417058, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:19,789 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36179,1732532417058/6bfc37bad115%2C36179%2C1732532417058.meta.1732532419759.meta 2024-11-25T11:00:19,789 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:19,790 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:19,792 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T11:00:19,795 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T11:00:19,801 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T11:00:19,805 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T11:00:19,806 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:19,806 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T11:00:19,806 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T11:00:19,808 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T11:00:19,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T11:00:19,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:19,810 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:19,811 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T11:00:19,812 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T11:00:19,812 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:19,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:19,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T11:00:19,813 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T11:00:19,814 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:19,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:19,814 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T11:00:19,815 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T11:00:19,815 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:19,816 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:19,816 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T11:00:19,818 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:19,819 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:19,819 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T11:00:19,820 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T11:00:19,820 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T11:00:19,822 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T11:00:19,823 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68775523, jitterRate=0.02483515441417694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T11:00:19,824 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T11:00:19,828 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732532419807Writing region info on filesystem at 1732532419807Initializing all the Stores at 1732532419807Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532419808 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532419808Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532419808Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532419808Cleaning up temporary data from old regions at 1732532419820 (+12 ms)Running coprocessor post-open hooks at 1732532419824 (+4 ms)Region opened successfully at 1732532419827 (+3 ms) 2024-11-25T11:00:19,837 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732532419724 2024-11-25T11:00:19,855 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T11:00:19,855 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,36179,1732532417058 2024-11-25T11:00:19,855 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T11:00:19,857 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,36179,1732532417058, state=OPEN 2024-11-25T11:00:19,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:19,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:19,861 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:19,861 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:19,861 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6bfc37bad115,36179,1732532417058 2024-11-25T11:00:19,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:19,865 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:19,866 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T11:00:19,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,36179,1732532417058 in 308 msec 2024-11-25T11:00:19,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T11:00:19,873 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 789 msec 2024-11-25T11:00:19,874 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:19,874 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T11:00:19,907 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T11:00:19,909 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,36179,1732532417058, seqNum=-1] 2024-11-25T11:00:19,936 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T11:00:19,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58967, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T11:00:19,966 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2830 sec 2024-11-25T11:00:19,967 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732532419967, completionTime=-1 2024-11-25T11:00:19,970 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T11:00:19,971 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T11:00:20,006 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T11:00:20,006 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732532480006 2024-11-25T11:00:20,007 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732532540007 2024-11-25T11:00:20,007 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 36 msec 2024-11-25T11:00:20,024 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T11:00:20,034 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,43465,1732532416119-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:20,034 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,43465,1732532416119-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:20,034 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,43465,1732532416119-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:20,036 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6bfc37bad115:43465, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:20,037 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:20,040 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:20,046 DEBUG [master/6bfc37bad115:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T11:00:20,077 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.577sec 2024-11-25T11:00:20,079 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T11:00:20,081 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T11:00:20,082 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T11:00:20,083 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T11:00:20,083 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T11:00:20,084 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,43465,1732532416119-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T11:00:20,085 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,43465,1732532416119-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T11:00:20,089 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T11:00:20,090 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T11:00:20,090 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,43465,1732532416119-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:20,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39df5dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T11:00:20,137 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-25T11:00:20,137 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-25T11:00:20,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,43465,-1 for getting cluster id 2024-11-25T11:00:20,147 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T11:00:20,158 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b31e9512-4557-46bb-8739-7bf24bbe80df' 2024-11-25T11:00:20,162 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T11:00:20,162 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b31e9512-4557-46bb-8739-7bf24bbe80df" 2024-11-25T11:00:20,163 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd87526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T11:00:20,163 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,43465,-1] 2024-11-25T11:00:20,166 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T11:00:20,168 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:20,170 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55750, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T11:00:20,174 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffe7030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T11:00:20,174 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T11:00:20,186 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,36179,1732532417058, seqNum=-1] 2024-11-25T11:00:20,186 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T11:00:20,190 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T11:00:20,218 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6bfc37bad115,43465,1732532416119 2024-11-25T11:00:20,224 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T11:00:20,230 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,43465,1732532416119 2024-11-25T11:00:20,234 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3aa4b4d1 2024-11-25T11:00:20,235 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T11:00:20,262 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T11:00:20,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnRegionMove1732532420218', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T11:00:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 2024-11-25T11:00:20,291 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T11:00:20,295 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:20,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732532420218" procId is: 4 2024-11-25T11:00:20,296 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T11:00:20,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:20,320 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 272a3cc60ae78a8a4c784f110272ba07, NAME => 'testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732532420218', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:20,328 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:20,329 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(1722): Closing 272a3cc60ae78a8a4c784f110272ba07, disabling compactions & flushes 2024-11-25T11:00:20,329 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:20,329 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:20,329 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. after waiting 0 ms 2024-11-25T11:00:20,329 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:20,329 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:20,329 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732532420218-pool-0 {}] regionserver.HRegion(1676): Region close journal for 272a3cc60ae78a8a4c784f110272ba07: Waiting for close lock at 1732532420329Disabling compacts and flushes for region at 1732532420329Disabling writes for close at 1732532420329Writing region close event to WAL at 1732532420329Closed at 1732532420329 2024-11-25T11:00:20,331 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T11:00:20,337 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732532420331"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532420331"}]},"ts":"1732532420331"} 2024-11-25T11:00:20,342 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T11:00:20,344 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T11:00:20,346 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732532420218","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532420344"}]},"ts":"1732532420344"} 2024-11-25T11:00:20,351 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732532420218, state=ENABLING in hbase:meta 2024-11-25T11:00:20,352 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {6bfc37bad115=0} racks are {/default-rack=0} 2024-11-25T11:00:20,352 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T11:00:20,352 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T11:00:20,352 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T11:00:20,352 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T11:00:20,352 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T11:00:20,352 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T11:00:20,352 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T11:00:20,354 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, ASSIGN}] 2024-11-25T11:00:20,357 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, ASSIGN 2024-11-25T11:00:20,359 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, ASSIGN; state=OFFLINE, location=6bfc37bad115,36179,1732532417058; forceNewPlan=false, retain=false 2024-11-25T11:00:20,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:20,511 INFO [6bfc37bad115:43465 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T11:00:20,513 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=272a3cc60ae78a8a4c784f110272ba07, regionState=OPENING, regionLocation=6bfc37bad115,36179,1732532417058 2024-11-25T11:00:20,520 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, ASSIGN because future has completed 2024-11-25T11:00:20,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,36179,1732532417058}] 2024-11-25T11:00:20,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:20,680 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:20,680 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 272a3cc60ae78a8a4c784f110272ba07, NAME => 'testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:20,681 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732532420218 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,681 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:20,681 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,681 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,682 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,687 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 272a3cc60ae78a8a4c784f110272ba07 columnFamilyName family 2024-11-25T11:00:20,688 DEBUG [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:20,688 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.HStore(327): Store=272a3cc60ae78a8a4c784f110272ba07/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:20,692 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,693 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,693 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,694 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,694 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,695 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,703 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T11:00:20,703 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 272a3cc60ae78a8a4c784f110272ba07; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75214720, jitterRate=0.12078666687011719}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T11:00:20,704 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:20,705 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 272a3cc60ae78a8a4c784f110272ba07: Running coprocessor pre-open hook at 1732532420681Writing region info on filesystem at 1732532420681Initializing all the Stores at 1732532420681Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532420682 (+1 ms)Cleaning up temporary data from old regions at 1732532420694 (+12 ms)Running coprocessor post-open hooks at 1732532420704 (+10 ms)Region opened successfully at 1732532420704 2024-11-25T11:00:20,706 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07., pid=6, masterSystemTime=1732532420676 2024-11-25T11:00:20,711 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:20,711 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:20,711 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=272a3cc60ae78a8a4c784f110272ba07, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,36179,1732532417058 2024-11-25T11:00:20,716 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,36179,1732532417058 because future has completed 2024-11-25T11:00:20,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T11:00:20,721 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,36179,1732532417058 in 196 msec 2024-11-25T11:00:20,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T11:00:20,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, ASSIGN in 367 msec 2024-11-25T11:00:20,725 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T11:00:20,725 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732532420218","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532420725"}]},"ts":"1732532420725"} 2024-11-25T11:00:20,728 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732532420218, state=ENABLED in hbase:meta 2024-11-25T11:00:20,729 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T11:00:20,732 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732532420218 in 455 msec 2024-11-25T11:00:20,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:20,937 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732532420218 completed 2024-11-25T11:00:20,937 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732532420218 get assigned. Timeout = 60000ms 2024-11-25T11:00:20,941 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T11:00:20,951 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732532420218 assigned to meta. Checking AM states. 2024-11-25T11:00:20,952 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T11:00:20,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732532420218 assigned. 2024-11-25T11:00:20,964 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732532420218', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07., hostname=6bfc37bad115,36179,1732532417058, seqNum=2] 2024-11-25T11:00:20,985 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 272a3cc60ae78a8a4c784f110272ba07 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T11:00:21,037 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 4136bf264a8840329e257c35a224682f 2024-11-25T11:00:21,108 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/.tmp/family/4136bf264a8840329e257c35a224682f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T11:00:21,117 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/.tmp/family/4136bf264a8840329e257c35a224682f 2024-11-25T11:00:21,162 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/.tmp/family/4136bf264a8840329e257c35a224682f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f 2024-11-25T11:00:21,163 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f, delay=916 ms 2024-11-25T11:00:21,166 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T11:00:21,173 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 272a3cc60ae78a8a4c784f110272ba07 in 188ms, sequenceid=6, compaction requested=false 2024-11-25T11:00:21,174 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732532420218' 2024-11-25T11:00:21,176 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 272a3cc60ae78a8a4c784f110272ba07: 2024-11-25T11:00:22,082 DEBUG [hfile-prefetch-1732532421163 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4136bf264a8840329e257c35a224682f_0. 2024-11-25T11:00:22,088 DEBUG [hfile-prefetch-1732532421163 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 4136bf264a8840329e257c35a224682f_117. 2024-11-25T11:00:22,089 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f, with total blocks 2, and data blocks 1 2024-11-25T11:00:22,089 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T11:00:22,090 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T11:00:22,090 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f. Will sleep 100ms and try the verification again. 2024-11-25T11:00:22,163 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T11:00:22,163 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T11:00:22,164 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:22,164 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:22,179 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T11:00:22,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T11:00:22,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:22,179 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:22,183 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T11:00:22,183 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732532422163 2024-11-25T11:00:22,183 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T11:00:22,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=272a3cc60ae78a8a4c784f110272ba07, source=6bfc37bad115,36179,1732532417058, destination=6bfc37bad115,35919,1732532417275, warming up region on 6bfc37bad115,35919,1732532417275 2024-11-25T11:00:22,191 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f, with total blocks 2, and data blocks 1 2024-11-25T11:00:22,191 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-25T11:00:22,191 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2293): found block for file 4136bf264a8840329e257c35a224682f in the backing map. Acquiring read lock for offset 0 2024-11-25T11:00:22,191 DEBUG [hfile-prefetch-1732532421163 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f has now been fully cached. 2024-11-25T11:00:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T11:00:22,192 DEBUG [hfile-prefetch-1732532421163 {}] hfile.PrefetchExecutor(121): Prefetch completed for 4136bf264a8840329e257c35a224682f 2024-11-25T11:00:22,193 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-25T11:00:22,193 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732532422179 2024-11-25T11:00:22,193 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T11:00:22,195 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37975, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T11:00:22,195 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=272a3cc60ae78a8a4c784f110272ba07, source=6bfc37bad115,36179,1732532417058, destination=6bfc37bad115,35919,1732532417275, running balancer 2024-11-25T11:00:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, REOPEN/MOVE 2024-11-25T11:00:22,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(7855): Warmup {ENCODED => 272a3cc60ae78a8a4c784f110272ba07, NAME => 'testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:22,197 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, REOPEN/MOVE 2024-11-25T11:00:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:22,198 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,200 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 272a3cc60ae78a8a4c784f110272ba07 columnFamilyName family 2024-11-25T11:00:22,200 DEBUG [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:22,200 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=272a3cc60ae78a8a4c784f110272ba07, regionState=CLOSING, regionLocation=6bfc37bad115,36179,1732532417058 2024-11-25T11:00:22,204 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, REOPEN/MOVE because future has completed 2024-11-25T11:00:22,204 DEBUG [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f 2024-11-25T11:00:22,204 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-25T11:00:22,205 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.HStore(327): Store=272a3cc60ae78a8a4c784f110272ba07/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:22,205 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,36179,1732532417058}] 2024-11-25T11:00:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(1722): Closing 272a3cc60ae78a8a4c784f110272ba07, disabling compactions & flushes 2024-11-25T11:00:22,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. after waiting 0 ms 2024-11-25T11:00:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35919 {}] regionserver.HRegion(1676): Region close journal for 272a3cc60ae78a8a4c784f110272ba07: Waiting for close lock at 1732532422205Disabling compacts and flushes for region at 1732532422205Disabling writes for close at 1732532422205Writing region close event to WAL at 1732532422207 (+2 ms)Closed at 1732532422207 2024-11-25T11:00:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43465 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-25T11:00:22,365 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,365 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-25T11:00:22,366 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 272a3cc60ae78a8a4c784f110272ba07, disabling compactions & flushes 2024-11-25T11:00:22,366 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,366 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,366 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. after waiting 0 ms 2024-11-25T11:00:22,366 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,370 DEBUG [StoreFileCloser-272a3cc60ae78a8a4c784f110272ba07-family-1 {}] bucket.BucketCache(768): Removing file 4136bf264a8840329e257c35a224682f for region 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,371 DEBUG [StoreFileCloser-272a3cc60ae78a8a4c784f110272ba07-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 4136bf264a8840329e257c35a224682f, starting offset: 0, end offset: 9223372036854775807 2024-11-25T11:00:22,372 DEBUG [StoreFileCloser-272a3cc60ae78a8a4c784f110272ba07-family-1 {}] bucket.BucketCache(856): removed key 4136bf264a8840329e257c35a224682f_0 from back map with offset lock 2094080 in the evict process 2024-11-25T11:00:22,382 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T11:00:22,383 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T11:00:22,384 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T11:00:22,384 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,384 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:22,384 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 272a3cc60ae78a8a4c784f110272ba07: Waiting for close lock at 1732532422366Running coprocessor pre-close hooks at 1732532422366Disabling compacts and flushes for region at 1732532422366Disabling writes for close at 1732532422366Writing region close event to WAL at 1732532422373 (+7 ms)Running coprocessor post-close hooks at 1732532422382 (+9 ms)Closed at 1732532422383 (+1 ms) 2024-11-25T11:00:22,384 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:22,384 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 272a3cc60ae78a8a4c784f110272ba07 move to 6bfc37bad115,35919,1732532417275 record at close sequenceid=6 2024-11-25T11:00:22,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=272a3cc60ae78a8a4c784f110272ba07, regionState=CLOSED 2024-11-25T11:00:22,395 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,396 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T11:00:22,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T11:00:22,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T11:00:22,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732532422384 2024-11-25T11:00:22,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T11:00:22,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:22,396 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:22,398 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,36179,1732532417058 because future has completed 2024-11-25T11:00:22,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T11:00:22,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732532422396 2024-11-25T11:00:22,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T11:00:22,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T11:00:22,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,36179,1732532417058 in 204 msec 2024-11-25T11:00:22,417 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, REOPEN/MOVE; state=CLOSED, location=6bfc37bad115,35919,1732532417275; forceNewPlan=false, retain=false 2024-11-25T11:00:22,567 INFO [6bfc37bad115:43465 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T11:00:22,568 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=272a3cc60ae78a8a4c784f110272ba07, regionState=OPENING, regionLocation=6bfc37bad115,35919,1732532417275 2024-11-25T11:00:22,571 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, REOPEN/MOVE because future has completed 2024-11-25T11:00:22,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,35919,1732532417275}] 2024-11-25T11:00:22,727 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,728 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 272a3cc60ae78a8a4c784f110272ba07, NAME => 'testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:22,728 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732532420218 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,729 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:22,729 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,729 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,730 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,731 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 272a3cc60ae78a8a4c784f110272ba07 columnFamilyName family 2024-11-25T11:00:22,731 DEBUG [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:22,734 DEBUG [StoreFileOpener-272a3cc60ae78a8a4c784f110272ba07-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f, delay=932 ms 2024-11-25T11:00:22,736 DEBUG [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f 2024-11-25T11:00:22,736 INFO [StoreOpener-272a3cc60ae78a8a4c784f110272ba07-1 {}] regionserver.HStore(327): Store=272a3cc60ae78a8a4c784f110272ba07/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:22,736 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,738 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,738 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,738 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,738 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,745 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,745 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 272a3cc60ae78a8a4c784f110272ba07; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66739105, jitterRate=-0.0055098384618759155}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T11:00:22,746 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,747 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 272a3cc60ae78a8a4c784f110272ba07: Running coprocessor pre-open hook at 1732532422729Writing region info on filesystem at 1732532422729Initializing all the Stores at 1732532422729Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532422730 (+1 ms)Cleaning up temporary data from old regions at 1732532422738 (+8 ms)Running coprocessor post-open hooks at 1732532422746 (+8 ms)Region opened successfully at 1732532422747 (+1 ms) 2024-11-25T11:00:22,748 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07., pid=9, masterSystemTime=1732532422724 2024-11-25T11:00:22,751 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,751 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,752 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=272a3cc60ae78a8a4c784f110272ba07, regionState=OPEN, openSeqNum=10, regionLocation=6bfc37bad115,35919,1732532417275 2024-11-25T11:00:22,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,35919,1732532417275 because future has completed 2024-11-25T11:00:22,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-25T11:00:22,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 272a3cc60ae78a8a4c784f110272ba07, server=6bfc37bad115,35919,1732532417275 in 185 msec 2024-11-25T11:00:22,761 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732532420218, region=272a3cc60ae78a8a4c784f110272ba07, REOPEN/MOVE in 564 msec 2024-11-25T11:00:22,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T11:00:22,801 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T11:00:22,801 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:22,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:22,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:22,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T11:00:22,809 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T11:00:22,809 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1861646758, stopped=false 2024-11-25T11:00:22,809 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6bfc37bad115,43465,1732532416119 2024-11-25T11:00:22,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:22,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:22,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:22,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:22,812 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T11:00:22,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:22,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:22,812 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T11:00:22,813 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:22,813 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:22,813 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:22,813 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:22,813 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:22,813 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6bfc37bad115,36179,1732532417058' ***** 2024-11-25T11:00:22,814 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T11:00:22,814 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6bfc37bad115,35919,1732532417275' ***** 2024-11-25T11:00:22,814 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T11:00:22,814 INFO [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T11:00:22,814 INFO [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T11:00:22,814 INFO [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.persistence 2024-11-25T11:00:22,814 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T11:00:22,814 INFO [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/bucket.persistence 2024-11-25T11:00:22,814 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T11:00:22,815 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T11:00:22,815 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T11:00:22,815 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T11:00:22,815 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T11:00:22,815 INFO [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T11:00:22,815 INFO [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T11:00:22,815 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T11:00:22,815 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T11:00:22,825 DEBUG [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1487): Thread RS:1;6bfc37bad115:35919 started persisting bucket cache to file 2024-11-25T11:00:22,825 DEBUG [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1487): Thread RS:0;6bfc37bad115:36179 started persisting bucket cache to file 2024-11-25T11:00:22,825 DEBUG [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:22,825 DEBUG [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:22,825 DEBUG [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:22,826 DEBUG [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:22,830 DEBUG [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T11:00:22,830 DEBUG [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T11:00:22,830 DEBUG [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732532422825 2024-11-25T11:00:22,830 DEBUG [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732532422825 2024-11-25T11:00:22,830 DEBUG [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1509): Thread RS:0;6bfc37bad115:36179 finished persisting bucket cache to file, renaming 2024-11-25T11:00:22,830 DEBUG [RS:1;6bfc37bad115:35919 {}] bucket.BucketCache(1509): Thread RS:1;6bfc37bad115:35919 finished persisting bucket cache to file, renaming 2024-11-25T11:00:22,830 WARN [RS:0;6bfc37bad115:36179 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-25T11:00:22,830 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T11:00:22,830 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T11:00:22,831 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T11:00:22,831 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T11:00:22,831 INFO [RS:0;6bfc37bad115:36179 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T11:00:22,831 INFO [RS:1;6bfc37bad115:35919 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T11:00:22,831 INFO [RS:1;6bfc37bad115:35919 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T11:00:22,831 INFO [RS:0;6bfc37bad115:36179 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T11:00:22,831 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(959): stopping server 6bfc37bad115,36179,1732532417058 2024-11-25T11:00:22,831 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(3091): Received CLOSE for 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,831 INFO [RS:0;6bfc37bad115:36179 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T11:00:22,831 INFO [RS:0;6bfc37bad115:36179 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6bfc37bad115:36179. 2024-11-25T11:00:22,831 DEBUG [RS:0;6bfc37bad115:36179 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:22,831 DEBUG [RS:0;6bfc37bad115:36179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:22,831 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T11:00:22,832 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T11:00:22,832 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T11:00:22,832 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T11:00:22,832 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(959): stopping server 6bfc37bad115,35919,1732532417275 2024-11-25T11:00:22,832 INFO [RS:1;6bfc37bad115:35919 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T11:00:22,832 INFO [RS:1;6bfc37bad115:35919 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6bfc37bad115:35919. 2024-11-25T11:00:22,832 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T11:00:22,832 DEBUG [RS:1;6bfc37bad115:35919 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:22,832 DEBUG [RS:1;6bfc37bad115:35919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:22,832 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-25T11:00:22,832 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-25T11:00:22,833 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1325): Online Regions={272a3cc60ae78a8a4c784f110272ba07=testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07.} 2024-11-25T11:00:22,833 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T11:00:22,833 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T11:00:22,833 DEBUG [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 272a3cc60ae78a8a4c784f110272ba07, disabling compactions & flushes 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T11:00:22,833 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,833 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T11:00:22,833 DEBUG [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1351): Waiting on 272a3cc60ae78a8a4c784f110272ba07 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. after waiting 0 ms 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T11:00:22,833 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T11:00:22,834 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-25T11:00:22,834 DEBUG [StoreFileCloser-272a3cc60ae78a8a4c784f110272ba07-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/family/4136bf264a8840329e257c35a224682f 2024-11-25T11:00:22,840 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnRegionMove1732532420218/272a3cc60ae78a8a4c784f110272ba07/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T11:00:22,840 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,840 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 272a3cc60ae78a8a4c784f110272ba07: Waiting for close lock at 1732532422832Running coprocessor pre-close hooks at 1732532422833 (+1 ms)Disabling compacts and flushes for region at 1732532422833Disabling writes for close at 1732532422833Writing region close event to WAL at 1732532422834 (+1 ms)Running coprocessor post-close hooks at 1732532422840 (+6 ms)Closed at 1732532422840 2024-11-25T11:00:22,841 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07. 2024-11-25T11:00:22,846 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on e55067165a394f5a8c46fb6542da63cc 2024-11-25T11:00:22,866 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/e55067165a394f5a8c46fb6542da63cc is 205, key is testBlockEvictionOnRegionMove1732532420218,,1732532420264.272a3cc60ae78a8a4c784f110272ba07./info:regioninfo/1732532422751/Put/seqid=0 2024-11-25T11:00:22,868 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/e55067165a394f5a8c46fb6542da63cc 2024-11-25T11:00:22,881 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on bcc9d13b45354dbabc306a0b305d914b 2024-11-25T11:00:22,898 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/ns/bcc9d13b45354dbabc306a0b305d914b is 43, key is default/ns:d/1732532419945/Put/seqid=0 2024-11-25T11:00:22,899 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/ns/bcc9d13b45354dbabc306a0b305d914b 2024-11-25T11:00:22,910 INFO [regionserver/6bfc37bad115:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:22,913 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7c9f083ec6d24de189e3e17aae73d4c5 2024-11-25T11:00:22,913 INFO [regionserver/6bfc37bad115:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:22,927 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/table/7c9f083ec6d24de189e3e17aae73d4c5 is 78, key is testBlockEvictionOnRegionMove1732532420218/table:state/1732532420725/Put/seqid=0 2024-11-25T11:00:22,928 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/table/7c9f083ec6d24de189e3e17aae73d4c5 2024-11-25T11:00:22,931 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/e55067165a394f5a8c46fb6542da63cc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/e55067165a394f5a8c46fb6542da63cc 2024-11-25T11:00:22,932 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/e55067165a394f5a8c46fb6542da63cc, delay=903 ms 2024-11-25T11:00:22,935 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/e55067165a394f5a8c46fb6542da63cc, entries=15, sequenceid=15, filesize=7.6 K 2024-11-25T11:00:22,935 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/ns/bcc9d13b45354dbabc306a0b305d914b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/bcc9d13b45354dbabc306a0b305d914b 2024-11-25T11:00:22,936 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/bcc9d13b45354dbabc306a0b305d914b, delay=997 ms 2024-11-25T11:00:22,938 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/bcc9d13b45354dbabc306a0b305d914b, entries=2, sequenceid=15, filesize=5.0 K 2024-11-25T11:00:22,938 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/table/7c9f083ec6d24de189e3e17aae73d4c5 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/7c9f083ec6d24de189e3e17aae73d4c5 2024-11-25T11:00:22,940 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/7c9f083ec6d24de189e3e17aae73d4c5, delay=913 ms 2024-11-25T11:00:22,941 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/7c9f083ec6d24de189e3e17aae73d4c5, entries=2, sequenceid=15, filesize=5.3 K 2024-11-25T11:00:22,942 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=15, compaction requested=false 2024-11-25T11:00:22,942 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-25T11:00:22,945 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/e55067165a394f5a8c46fb6542da63cc 2024-11-25T11:00:22,945 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/bcc9d13b45354dbabc306a0b305d914b 2024-11-25T11:00:22,946 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/7c9f083ec6d24de189e3e17aae73d4c5 2024-11-25T11:00:22,953 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-25T11:00:22,953 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T11:00:22,954 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:22,954 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732532422832Running coprocessor pre-close hooks at 1732532422833 (+1 ms)Disabling compacts and flushes for region at 1732532422833Disabling writes for close at 1732532422833Obtaining lock to block concurrent updates at 1732532422834 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732532422834Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732532422834Flushing stores of hbase:meta,,1.1588230740 at 1732532422834Flushing 1588230740/info: creating writer at 1732532422835 (+1 ms)Flushing 1588230740/info: appending metadata at 1732532422864 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732532422865 (+1 ms)Flushing 1588230740/ns: creating writer at 1732532422872 (+7 ms)Flushing 1588230740/ns: appending metadata at 1732532422896 (+24 ms)Flushing 1588230740/ns: closing flushed file at 1732532422896Flushing 1588230740/table: creating writer at 1732532422902 (+6 ms)Flushing 1588230740/table: appending metadata at 1732532422926 (+24 ms)Flushing 1588230740/table: closing flushed file at 1732532422926Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@279b2029: reopening flushed file at 1732532422931 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57ce5151: reopening flushed file at 1732532422935 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75c1748e: reopening flushed file at 1732532422938 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 109ms, sequenceid=15, compaction requested=false at 1732532422942 (+4 ms)Writing region close event to WAL at 1732532422947 (+5 ms)Running coprocessor post-close hooks at 1732532422953 (+6 ms)Closed at 1732532422954 (+1 ms) 2024-11-25T11:00:22,954 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:23,033 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(976): stopping server 6bfc37bad115,36179,1732532417058; all regions closed. 2024-11-25T11:00:23,033 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(976): stopping server 6bfc37bad115,35919,1732532417275; all regions closed. 2024-11-25T11:00:23,040 DEBUG [RS:1;6bfc37bad115:35919 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:23,044 INFO [RS:1;6bfc37bad115:35919 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C35919%2C1732532417275:(num 1732532419301) 2024-11-25T11:00:23,044 DEBUG [RS:1;6bfc37bad115:35919 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:23,044 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:23,044 INFO [RS:1;6bfc37bad115:35919 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T11:00:23,045 INFO [RS:1;6bfc37bad115:35919 {}] hbase.ChoreService(370): Chore service for: regionserver/6bfc37bad115:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T11:00:23,045 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T11:00:23,045 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T11:00:23,045 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T11:00:23,045 INFO [RS:1;6bfc37bad115:35919 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T11:00:23,045 INFO [regionserver/6bfc37bad115:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T11:00:23,045 INFO [RS:1;6bfc37bad115:35919 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35919 2024-11-25T11:00:23,050 DEBUG [RS:0;6bfc37bad115:36179 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:23,050 INFO [RS:0;6bfc37bad115:36179 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C36179%2C1732532417058.meta:.meta(num 1732532419759) 2024-11-25T11:00:23,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:23,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6bfc37bad115,35919,1732532417275 2024-11-25T11:00:23,052 INFO [RS:1;6bfc37bad115:35919 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T11:00:23,054 DEBUG [RS:0;6bfc37bad115:36179 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:23,054 INFO [RS:0;6bfc37bad115:36179 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C36179%2C1732532417058:(num 1732532419304) 2024-11-25T11:00:23,054 DEBUG [RS:0;6bfc37bad115:36179 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:23,054 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:23,054 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6bfc37bad115,35919,1732532417275] 2024-11-25T11:00:23,054 INFO [RS:0;6bfc37bad115:36179 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T11:00:23,054 INFO [RS:0;6bfc37bad115:36179 {}] hbase.ChoreService(370): Chore service for: regionserver/6bfc37bad115:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T11:00:23,055 INFO [RS:0;6bfc37bad115:36179 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T11:00:23,055 INFO [regionserver/6bfc37bad115:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T11:00:23,056 INFO [RS:0;6bfc37bad115:36179 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36179 2024-11-25T11:00:23,056 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6bfc37bad115,35919,1732532417275 already deleted, retry=false 2024-11-25T11:00:23,057 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6bfc37bad115,35919,1732532417275 expired; onlineServers=1 2024-11-25T11:00:23,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6bfc37bad115,36179,1732532417058 2024-11-25T11:00:23,059 INFO [RS:0;6bfc37bad115:36179 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T11:00:23,065 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:23,065 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6bfc37bad115,36179,1732532417058] 2024-11-25T11:00:23,067 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6bfc37bad115,36179,1732532417058 already deleted, retry=false 2024-11-25T11:00:23,067 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6bfc37bad115,36179,1732532417058 expired; onlineServers=0 2024-11-25T11:00:23,067 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6bfc37bad115,43465,1732532416119' ***** 2024-11-25T11:00:23,067 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T11:00:23,067 INFO [M:0;6bfc37bad115:43465 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T11:00:23,067 INFO [M:0;6bfc37bad115:43465 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T11:00:23,068 DEBUG [M:0;6bfc37bad115:43465 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T11:00:23,068 DEBUG [M:0;6bfc37bad115:43465 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T11:00:23,068 DEBUG [master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532418838 {}] cleaner.HFileCleaner(306): Exit Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532418838,5,FailOnTimeoutGroup] 2024-11-25T11:00:23,068 DEBUG [master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532418841 {}] cleaner.HFileCleaner(306): Exit Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532418841,5,FailOnTimeoutGroup] 2024-11-25T11:00:23,068 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T11:00:23,069 INFO [M:0;6bfc37bad115:43465 {}] hbase.ChoreService(370): Chore service for: master/6bfc37bad115:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T11:00:23,069 INFO [M:0;6bfc37bad115:43465 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T11:00:23,069 DEBUG [M:0;6bfc37bad115:43465 {}] master.HMaster(1795): Stopping service threads 2024-11-25T11:00:23,069 INFO [M:0;6bfc37bad115:43465 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T11:00:23,069 INFO [M:0;6bfc37bad115:43465 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T11:00:23,070 INFO [M:0;6bfc37bad115:43465 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T11:00:23,070 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T11:00:23,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T11:00:23,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,073 DEBUG [M:0;6bfc37bad115:43465 {}] zookeeper.ZKUtil(347): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-25T11:00:23,073 WARN [M:0;6bfc37bad115:43465 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-25T11:00:23,073 INFO [M:0;6bfc37bad115:43465 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.lastflushedseqids 2024-11-25T11:00:23,094 INFO [M:0;6bfc37bad115:43465 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T11:00:23,094 INFO [M:0;6bfc37bad115:43465 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T11:00:23,094 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T11:00:23,094 INFO [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,094 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,094 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T11:00:23,095 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,095 INFO [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-25T11:00:23,110 DEBUG [M:0;6bfc37bad115:43465 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 235d6850cc714960b23c9eb57cbac1e7 2024-11-25T11:00:23,124 DEBUG [M:0;6bfc37bad115:43465 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/235d6850cc714960b23c9eb57cbac1e7 is 82, key is hbase:meta,,1/info:regioninfo/1732532419854/Put/seqid=0 2024-11-25T11:00:23,125 INFO [M:0;6bfc37bad115:43465 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/235d6850cc714960b23c9eb57cbac1e7 2024-11-25T11:00:23,135 DEBUG [M:0;6bfc37bad115:43465 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3cf3f6b658c147068f25d48c4239d0c8 2024-11-25T11:00:23,151 DEBUG [M:0;6bfc37bad115:43465 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3cf3f6b658c147068f25d48c4239d0c8 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732532420731/Put/seqid=0 2024-11-25T11:00:23,152 INFO [M:0;6bfc37bad115:43465 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3cf3f6b658c147068f25d48c4239d0c8 2024-11-25T11:00:23,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:23,158 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35919-0x1007698bf5e0002, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:23,158 INFO [RS:1;6bfc37bad115:35919 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T11:00:23,159 INFO [RS:1;6bfc37bad115:35919 {}] regionserver.HRegionServer(1031): Exiting; stopping=6bfc37bad115,35919,1732532417275; zookeeper connection closed. 2024-11-25T11:00:23,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:23,161 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36179-0x1007698bf5e0001, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:23,161 INFO [RS:0;6bfc37bad115:36179 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T11:00:23,161 INFO [RS:0;6bfc37bad115:36179 {}] regionserver.HRegionServer(1031): Exiting; stopping=6bfc37bad115,36179,1732532417058; zookeeper connection closed. 2024-11-25T11:00:23,161 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1f940e05 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1f940e05 2024-11-25T11:00:23,161 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@71fd0ec8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@71fd0ec8 2024-11-25T11:00:23,162 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-25T11:00:23,171 DEBUG [M:0;6bfc37bad115:43465 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7faaec9d640d45569c68fe44c5417772 2024-11-25T11:00:23,188 DEBUG [M:0;6bfc37bad115:43465 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7faaec9d640d45569c68fe44c5417772 is 69, key is 6bfc37bad115,35919,1732532417275/rs:state/1732532418909/Put/seqid=0 2024-11-25T11:00:23,188 INFO [M:0;6bfc37bad115:43465 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7faaec9d640d45569c68fe44c5417772 2024-11-25T11:00:23,191 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/235d6850cc714960b23c9eb57cbac1e7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/235d6850cc714960b23c9eb57cbac1e7 2024-11-25T11:00:23,193 INFO [M:0;6bfc37bad115:43465 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/235d6850cc714960b23c9eb57cbac1e7, entries=8, sequenceid=84, filesize=5.5 K 2024-11-25T11:00:23,193 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3cf3f6b658c147068f25d48c4239d0c8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3cf3f6b658c147068f25d48c4239d0c8 2024-11-25T11:00:23,195 INFO [M:0;6bfc37bad115:43465 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3cf3f6b658c147068f25d48c4239d0c8, entries=9, sequenceid=84, filesize=6.5 K 2024-11-25T11:00:23,196 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7faaec9d640d45569c68fe44c5417772 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7faaec9d640d45569c68fe44c5417772 2024-11-25T11:00:23,198 INFO [M:0;6bfc37bad115:43465 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7faaec9d640d45569c68fe44c5417772, entries=2, sequenceid=84, filesize=5.1 K 2024-11-25T11:00:23,199 INFO [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=84, compaction requested=false 2024-11-25T11:00:23,205 INFO [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,205 DEBUG [M:0;6bfc37bad115:43465 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732532423094Disabling compacts and flushes for region at 1732532423094Disabling writes for close at 1732532423095 (+1 ms)Obtaining lock to block concurrent updates at 1732532423095Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732532423095Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732532423095Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732532423096 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732532423096Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732532423124 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732532423124Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732532423127 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732532423150 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732532423151 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732532423154 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732532423187 (+33 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732532423187Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d1f7811: reopening flushed file at 1732532423190 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b0ef340: reopening flushed file at 1732532423193 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77cc56d7: reopening flushed file at 1732532423196 (+3 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 104ms, sequenceid=84, compaction requested=false at 1732532423199 (+3 ms)Writing region close event to WAL at 1732532423205 (+6 ms)Closed at 1732532423205 2024-11-25T11:00:23,208 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T11:00:23,208 INFO [M:0;6bfc37bad115:43465 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T11:00:23,208 INFO [M:0;6bfc37bad115:43465 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43465 2024-11-25T11:00:23,208 INFO [M:0;6bfc37bad115:43465 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T11:00:23,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:23,311 INFO [M:0;6bfc37bad115:43465 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T11:00:23,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:43465-0x1007698bf5e0000, quorum=127.0.0.1:55243, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:23,318 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T11:00:23,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T11:00:23,348 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,359 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6bfc37bad115:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732532422734 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: regionserver/6bfc37bad115:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732532421163 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732532422932 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6bfc37bad115:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732532422936 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6bfc37bad115:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=867 (was 907), ProcessCount=11 (was 11), AvailableMemoryMB=4513 (was 4449) - AvailableMemoryMB LEAK? - 2024-11-25T11:00:23,368 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=867, ProcessCount=11, AvailableMemoryMB=4512 2024-11-25T11:00:23,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/hadoop-log-dir so I do NOT create it in target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d 2024-11-25T11:00:23,369 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/hadoop-tmp-dir so I do NOT create it in target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d 2024-11-25T11:00:23,369 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,374 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/cluster_fe7c5b05-c11a-b49d-70d6-375eaeca8377, deleteOnExit=true 2024-11-25T11:00:23,382 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/cluster_fe7c5b05-c11a-b49d-70d6-375eaeca8377/zookeeper_0, clientPort=61363, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/cluster_fe7c5b05-c11a-b49d-70d6-375eaeca8377/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/cluster_fe7c5b05-c11a-b49d-70d6-375eaeca8377/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-25T11:00:23,385 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61363 2024-11-25T11:00:23,413 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d with version=8 2024-11-25T11:00:23,413 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/hbase-staging 2024-11-25T11:00:23,418 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T11:00:23,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,418 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,418 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T11:00:23,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,419 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T11:00:23,419 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-25T11:00:23,419 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T11:00:23,425 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39111 2024-11-25T11:00:23,427 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39111 connecting to ZooKeeper ensemble=127.0.0.1:61363 2024-11-25T11:00:23,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391110x0, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T11:00:23,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39111-0x1007698dd510000 connected 2024-11-25T11:00:23,514 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,522 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:23,522 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d, hbase.cluster.distributed=false 2024-11-25T11:00:23,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T11:00:23,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39111 2024-11-25T11:00:23,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39111 2024-11-25T11:00:23,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39111 2024-11-25T11:00:23,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39111 2024-11-25T11:00:23,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39111 2024-11-25T11:00:23,555 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T11:00:23,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,555 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,556 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T11:00:23,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,556 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T11:00:23,556 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T11:00:23,556 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T11:00:23,557 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38299 2024-11-25T11:00:23,559 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38299 connecting to ZooKeeper ensemble=127.0.0.1:61363 2024-11-25T11:00:23,559 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,560 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,566 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:382990x0, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T11:00:23,566 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38299-0x1007698dd510001 connected 2024-11-25T11:00:23,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:23,567 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T11:00:23,571 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.cache 2024-11-25T11:00:23,571 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T11:00:23,571 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T11:00:23,572 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T11:00:23,572 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T11:00:23,572 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T11:00:23,573 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T11:00:23,573 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence is complete. 2024-11-25T11:00:23,576 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T11:00:23,580 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T11:00:23,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T11:00:23,590 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T11:00:23,592 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38299 2024-11-25T11:00:23,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38299 2024-11-25T11:00:23,602 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38299 2024-11-25T11:00:23,602 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38299 2024-11-25T11:00:23,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38299 2024-11-25T11:00:23,631 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T11:00:23,631 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,631 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,631 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T11:00:23,631 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:23,631 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T11:00:23,631 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T11:00:23,632 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T11:00:23,637 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36685 2024-11-25T11:00:23,640 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36685 connecting to ZooKeeper ensemble=127.0.0.1:61363 2024-11-25T11:00:23,640 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,641 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,652 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:366850x0, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:23,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:366850x0, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T11:00:23,652 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T11:00:23,657 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36685-0x1007698dd510002 connected 2024-11-25T11:00:23,676 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.cache 2024-11-25T11:00:23,677 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T11:00:23,677 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T11:00:23,677 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T11:00:23,687 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T11:00:23,688 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T11:00:23,689 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-25T11:00:23,689 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T11:00:23,689 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence is complete. 2024-11-25T11:00:23,692 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T11:00:23,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-25T11:00:23,700 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T11:00:23,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36685 2024-11-25T11:00:23,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36685 2024-11-25T11:00:23,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36685 2024-11-25T11:00:23,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36685 2024-11-25T11:00:23,706 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36685 2024-11-25T11:00:23,722 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6bfc37bad115:39111 2024-11-25T11:00:23,722 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6bfc37bad115,39111,1732532423417 2024-11-25T11:00:23,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:23,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:23,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:23,728 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6bfc37bad115,39111,1732532423417 2024-11-25T11:00:23,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T11:00:23,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,732 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T11:00:23,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-25T11:00:23,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,734 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6bfc37bad115,39111,1732532423417 from backup master directory 2024-11-25T11:00:23,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:23,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6bfc37bad115,39111,1732532423417 2024-11-25T11:00:23,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:23,736 WARN [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T11:00:23,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-25T11:00:23,737 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6bfc37bad115,39111,1732532423417 2024-11-25T11:00:23,739 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/hbase.id] with ID: ba7d93d3-e485-4440-a168-037d33c3e62b 2024-11-25T11:00:23,739 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.tmp/hbase.id 2024-11-25T11:00:23,746 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/hbase.id] 2024-11-25T11:00:23,786 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:23,786 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-25T11:00:23,788 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-25T11:00:23,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,796 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T11:00:23,798 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-25T11:00:23,798 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:23,819 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store 2024-11-25T11:00:23,826 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:23,827 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T11:00:23,827 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,827 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,827 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T11:00:23,827 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,827 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:23,827 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732532423827Disabling compacts and flushes for region at 1732532423827Disabling writes for close at 1732532423827Writing region close event to WAL at 1732532423827Closed at 1732532423827 2024-11-25T11:00:23,828 WARN [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/.initializing 2024-11-25T11:00:23,828 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/WALs/6bfc37bad115,39111,1732532423417 2024-11-25T11:00:23,842 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T11:00:23,851 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6bfc37bad115%2C39111%2C1732532423417, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/WALs/6bfc37bad115,39111,1732532423417, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/oldWALs, maxLogs=10 2024-11-25T11:00:23,878 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/WALs/6bfc37bad115,39111,1732532423417/6bfc37bad115%2C39111%2C1732532423417.1732532423852 2024-11-25T11:00:23,879 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:23,879 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:23,880 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:23,880 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,880 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,885 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,899 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-25T11:00:23,899 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:23,900 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:23,900 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,908 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-25T11:00:23,908 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:23,909 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:23,909 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-25T11:00:23,924 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:23,924 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:23,925 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,937 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-25T11:00:23,937 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:23,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:23,938 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,939 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,939 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,939 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,939 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,940 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T11:00:23,940 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-25T11:00:23,951 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T11:00:23,951 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65807960, jitterRate=-0.01938498020172119}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T11:00:23,953 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732532423880Initializing all the Stores at 1732532423881 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532423881Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532423884 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532423884Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532423884Cleaning up temporary data from old regions at 1732532423939 (+55 ms)Region opened successfully at 1732532423952 (+13 ms) 2024-11-25T11:00:23,954 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-25T11:00:23,960 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@586b8869, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T11:00:23,963 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-25T11:00:23,964 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-25T11:00:23,964 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-25T11:00:23,964 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-25T11:00:23,965 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-25T11:00:23,965 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-25T11:00:23,965 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-25T11:00:23,975 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-25T11:00:23,976 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-25T11:00:23,977 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-25T11:00:23,978 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-25T11:00:23,978 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-25T11:00:23,980 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-25T11:00:23,980 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-25T11:00:23,985 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-25T11:00:23,986 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-25T11:00:23,987 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-25T11:00:23,989 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-25T11:00:23,991 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-25T11:00:23,992 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-25T11:00:23,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:23,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:23,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:23,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:23,997 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6bfc37bad115,39111,1732532423417, sessionid=0x1007698dd510000, setting cluster-up flag (Was=false) 2024-11-25T11:00:24,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,000 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,008 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-25T11:00:24,017 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6bfc37bad115,39111,1732532423417 2024-11-25T11:00:24,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,021 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,027 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-25T11:00:24,029 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6bfc37bad115,39111,1732532423417 2024-11-25T11:00:24,033 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-25T11:00:24,035 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:24,035 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-25T11:00:24,036 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-25T11:00:24,036 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6bfc37bad115,39111,1732532423417 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-25T11:00:24,048 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:24,048 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:24,048 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:24,049 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6bfc37bad115:0, corePoolSize=5, maxPoolSize=5 2024-11-25T11:00:24,049 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6bfc37bad115:0, corePoolSize=10, maxPoolSize=10 2024-11-25T11:00:24,049 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,049 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T11:00:24,049 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,050 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732532454050 2024-11-25T11:00:24,051 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:24,051 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-25T11:00:24,051 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-25T11:00:24,051 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-25T11:00:24,051 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-25T11:00:24,051 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-25T11:00:24,051 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-25T11:00:24,051 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-25T11:00:24,052 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,052 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-25T11:00:24,052 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,057 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-25T11:00:24,057 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-25T11:00:24,057 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-25T11:00:24,058 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-25T11:00:24,058 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-25T11:00:24,060 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532424058,5,FailOnTimeoutGroup] 2024-11-25T11:00:24,065 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532424061,5,FailOnTimeoutGroup] 2024-11-25T11:00:24,065 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,065 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-25T11:00:24,065 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,065 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,088 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-25T11:00:24,088 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:24,095 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:24,096 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T11:00:24,099 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T11:00:24,099 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,100 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,100 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T11:00:24,108 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T11:00:24,108 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T11:00:24,110 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(746): ClusterId : ba7d93d3-e485-4440-a168-037d33c3e62b 2024-11-25T11:00:24,110 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T11:00:24,113 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T11:00:24,113 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T11:00:24,118 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T11:00:24,118 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T11:00:24,119 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T11:00:24,122 DEBUG [RS:0;6bfc37bad115:38299 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a03b88a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T11:00:24,125 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T11:00:24,125 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,126 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,126 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T11:00:24,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:24,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:24,127 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(746): ClusterId : ba7d93d3-e485-4440-a168-037d33c3e62b 2024-11-25T11:00:24,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T11:00:24,127 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T11:00:24,127 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T11:00:24,128 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T11:00:24,128 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T11:00:24,135 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T11:00:24,135 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T11:00:24,139 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T11:00:24,139 DEBUG [RS:1;6bfc37bad115:36685 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@716d4ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T11:00:24,144 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T11:00:24,144 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64839034, jitterRate=-0.033823102712631226}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T11:00:24,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732532424095Initializing all the Stores at 1732532424095Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532424095Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532424095Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532424096 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532424096Cleaning up temporary data from old regions at 1732532424127 (+31 ms)Region opened successfully at 1732532424145 (+18 ms) 2024-11-25T11:00:24,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T11:00:24,145 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T11:00:24,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T11:00:24,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T11:00:24,145 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T11:00:24,149 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:24,149 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732532424145Disabling compacts and flushes for region at 1732532424145Disabling writes for close at 1732532424145Writing region close event to WAL at 1732532424149 (+4 ms)Closed at 1732532424149 2024-11-25T11:00:24,150 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6bfc37bad115:38299 2024-11-25T11:00:24,150 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T11:00:24,150 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T11:00:24,150 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T11:00:24,150 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:24,151 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-25T11:00:24,151 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T11:00:24,151 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(2659): reportForDuty to master=6bfc37bad115,39111,1732532423417 with port=38299, startcode=1732532423554 2024-11-25T11:00:24,152 DEBUG [RS:0;6bfc37bad115:38299 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T11:00:24,152 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T11:00:24,154 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-25T11:00:24,159 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6bfc37bad115:36685 2024-11-25T11:00:24,160 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T11:00:24,160 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T11:00:24,160 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T11:00:24,161 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(2659): reportForDuty to master=6bfc37bad115,39111,1732532423417 with port=36685, startcode=1732532423630 2024-11-25T11:00:24,162 DEBUG [RS:1;6bfc37bad115:36685 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T11:00:24,166 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T11:00:24,167 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39111 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,167 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39111 {}] master.ServerManager(517): Registering regionserver=6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,169 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:24,169 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T11:00:24,169 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T11:00:24,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:24,172 DEBUG [RS:0;6bfc37bad115:38299 {}] zookeeper.ZKUtil(111): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,172 WARN [RS:0;6bfc37bad115:38299 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T11:00:24,172 INFO [RS:0;6bfc37bad115:38299 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:24,173 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,174 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36253, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T11:00:24,174 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39111 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6bfc37bad115,36685,1732532423630 2024-11-25T11:00:24,175 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39111 {}] master.ServerManager(517): Registering regionserver=6bfc37bad115,36685,1732532423630 2024-11-25T11:00:24,176 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6bfc37bad115,38299,1732532423554] 2024-11-25T11:00:24,177 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:24,177 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T11:00:24,177 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T11:00:24,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:24,184 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6bfc37bad115,36685,1732532423630] 2024-11-25T11:00:24,184 DEBUG [RS:1;6bfc37bad115:36685 {}] zookeeper.ZKUtil(111): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6bfc37bad115,36685,1732532423630 2024-11-25T11:00:24,184 WARN [RS:1;6bfc37bad115:36685 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T11:00:24,184 INFO [RS:1;6bfc37bad115:36685 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:24,185 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36685,1732532423630 2024-11-25T11:00:24,188 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T11:00:24,191 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T11:00:24,192 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T11:00:24,192 INFO [RS:0;6bfc37bad115:38299 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T11:00:24,192 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,192 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T11:00:24,193 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T11:00:24,194 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,194 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,194 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,194 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,194 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,194 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,194 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,195 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:24,195 DEBUG [RS:0;6bfc37bad115:38299 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:24,196 INFO [RS:1;6bfc37bad115:36685 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T11:00:24,196 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,200 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T11:00:24,201 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T11:00:24,201 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,201 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,201 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:24,202 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:24,203 DEBUG [RS:1;6bfc37bad115:36685 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:24,209 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,209 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,209 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,209 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,209 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,209 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,38299,1732532423554-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T11:00:24,216 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,217 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,217 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,217 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,217 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,217 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,36685,1732532423630-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T11:00:24,234 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T11:00:24,234 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,38299,1732532423554-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,234 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,234 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.Replication(171): 6bfc37bad115,38299,1732532423554 started 2024-11-25T11:00:24,244 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T11:00:24,244 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,36685,1732532423630-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,244 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,244 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.Replication(171): 6bfc37bad115,36685,1732532423630 started 2024-11-25T11:00:24,258 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,258 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1482): Serving as 6bfc37bad115,38299,1732532423554, RpcServer on 6bfc37bad115/172.17.0.3:38299, sessionid=0x1007698dd510001 2024-11-25T11:00:24,259 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T11:00:24,259 DEBUG [RS:0;6bfc37bad115:38299 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,259 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,38299,1732532423554' 2024-11-25T11:00:24,259 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T11:00:24,260 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T11:00:24,261 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T11:00:24,261 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T11:00:24,261 DEBUG [RS:0;6bfc37bad115:38299 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,261 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,38299,1732532423554' 2024-11-25T11:00:24,261 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T11:00:24,262 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T11:00:24,264 DEBUG [RS:0;6bfc37bad115:38299 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T11:00:24,264 INFO [RS:0;6bfc37bad115:38299 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T11:00:24,264 INFO [RS:0;6bfc37bad115:38299 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T11:00:24,269 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,270 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1482): Serving as 6bfc37bad115,36685,1732532423630, RpcServer on 6bfc37bad115/172.17.0.3:36685, sessionid=0x1007698dd510002 2024-11-25T11:00:24,270 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T11:00:24,270 DEBUG [RS:1;6bfc37bad115:36685 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6bfc37bad115,36685,1732532423630 2024-11-25T11:00:24,270 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,36685,1732532423630' 2024-11-25T11:00:24,270 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T11:00:24,273 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T11:00:24,274 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T11:00:24,274 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T11:00:24,274 DEBUG [RS:1;6bfc37bad115:36685 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6bfc37bad115,36685,1732532423630 2024-11-25T11:00:24,274 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,36685,1732532423630' 2024-11-25T11:00:24,274 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T11:00:24,274 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T11:00:24,275 DEBUG [RS:1;6bfc37bad115:36685 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T11:00:24,275 INFO [RS:1;6bfc37bad115:36685 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T11:00:24,275 INFO [RS:1;6bfc37bad115:36685 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T11:00:24,305 WARN [6bfc37bad115:39111 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-25T11:00:24,365 INFO [RS:0;6bfc37bad115:38299 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T11:00:24,367 INFO [RS:0;6bfc37bad115:38299 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C38299%2C1732532423554, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:24,376 INFO [RS:1;6bfc37bad115:36685 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T11:00:24,376 INFO [RS:1;6bfc37bad115:36685 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C36685%2C1732532423630, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36685,1732532423630, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:24,401 INFO [RS:0;6bfc37bad115:38299 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554/6bfc37bad115%2C38299%2C1732532423554.1732532424369 2024-11-25T11:00:24,403 DEBUG [RS:0;6bfc37bad115:38299 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:24,404 INFO [RS:1;6bfc37bad115:36685 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36685,1732532423630/6bfc37bad115%2C36685%2C1732532423630.1732532424377 2024-11-25T11:00:24,404 DEBUG [RS:1;6bfc37bad115:36685 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:24,555 DEBUG [6bfc37bad115:39111 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-25T11:00:24,556 DEBUG [6bfc37bad115:39111 {}] balancer.BalancerClusterState(204): Hosts are {6bfc37bad115=0} racks are {/default-rack=0} 2024-11-25T11:00:24,557 DEBUG [6bfc37bad115:39111 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T11:00:24,557 DEBUG [6bfc37bad115:39111 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T11:00:24,557 DEBUG [6bfc37bad115:39111 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T11:00:24,557 DEBUG [6bfc37bad115:39111 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T11:00:24,557 INFO [6bfc37bad115:39111 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T11:00:24,557 INFO [6bfc37bad115:39111 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T11:00:24,557 DEBUG [6bfc37bad115:39111 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T11:00:24,557 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,559 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,38299,1732532423554, state=OPENING 2024-11-25T11:00:24,561 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-25T11:00:24,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,563 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:24,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:24,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:24,565 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:24,565 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T11:00:24,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,38299,1732532423554}] 2024-11-25T11:00:24,719 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T11:00:24,722 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47559, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T11:00:24,726 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T11:00:24,726 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:24,727 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T11:00:24,728 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C38299%2C1732532423554.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:24,761 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554/6bfc37bad115%2C38299%2C1732532423554.meta.1732532424729.meta 2024-11-25T11:00:24,764 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:24,765 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:24,765 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T11:00:24,765 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T11:00:24,766 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T11:00:24,766 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T11:00:24,766 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:24,766 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T11:00:24,766 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T11:00:24,769 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T11:00:24,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T11:00:24,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,770 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T11:00:24,770 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T11:00:24,771 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T11:00:24,771 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T11:00:24,772 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T11:00:24,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T11:00:24,773 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,773 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:24,778 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T11:00:24,780 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:24,781 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:24,781 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T11:00:24,781 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T11:00:24,781 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T11:00:24,783 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T11:00:24,783 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71183083, jitterRate=0.060710594058036804}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T11:00:24,783 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T11:00:24,786 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732532424766Writing region info on filesystem at 1732532424767 (+1 ms)Initializing all the Stores at 1732532424767Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532424767Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532424768 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532424768Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532424768Cleaning up temporary data from old regions at 1732532424781 (+13 ms)Running coprocessor post-open hooks at 1732532424783 (+2 ms)Region opened successfully at 1732532424784 (+1 ms) 2024-11-25T11:00:24,787 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732532424718 2024-11-25T11:00:24,793 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,794 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,38299,1732532423554, state=OPEN 2024-11-25T11:00:24,795 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T11:00:24,795 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T11:00:24,800 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6bfc37bad115,38299,1732532423554 2024-11-25T11:00:24,800 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:24,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:24,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:24,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:24,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:24,801 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:24,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-25T11:00:24,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,38299,1732532423554 in 235 msec 2024-11-25T11:00:24,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-25T11:00:24,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 652 msec 2024-11-25T11:00:24,810 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-25T11:00:24,810 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-25T11:00:24,822 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T11:00:24,823 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,38299,1732532423554, seqNum=-1] 2024-11-25T11:00:24,823 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T11:00:24,827 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44333, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T11:00:24,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 802 msec 2024-11-25T11:00:24,838 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732532424838, completionTime=-1 2024-11-25T11:00:24,838 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-25T11:00:24,838 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-25T11:00:24,843 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-25T11:00:24,843 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732532484843 2024-11-25T11:00:24,843 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732532544843 2024-11-25T11:00:24,843 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 4 msec 2024-11-25T11:00:24,843 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-25T11:00:24,844 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39111,1732532423417-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,844 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39111,1732532423417-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,844 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39111,1732532423417-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,844 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6bfc37bad115:39111, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,844 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,844 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,850 DEBUG [master/6bfc37bad115:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-25T11:00:24,853 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.115sec 2024-11-25T11:00:24,856 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-25T11:00:24,856 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-25T11:00:24,856 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-25T11:00:24,856 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-25T11:00:24,857 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-25T11:00:24,857 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39111,1732532423417-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T11:00:24,857 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39111,1732532423417-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-25T11:00:24,859 DEBUG [master/6bfc37bad115:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-25T11:00:24,859 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-25T11:00:24,859 INFO [master/6bfc37bad115:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,39111,1732532423417-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:24,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a912745, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T11:00:24,922 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6bfc37bad115,39111,-1 for getting cluster id 2024-11-25T11:00:24,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-25T11:00:24,927 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'ba7d93d3-e485-4440-a168-037d33c3e62b' 2024-11-25T11:00:24,929 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-25T11:00:24,929 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "ba7d93d3-e485-4440-a168-037d33c3e62b" 2024-11-25T11:00:24,930 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62bf8d9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T11:00:24,930 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6bfc37bad115,39111,-1] 2024-11-25T11:00:24,930 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-25T11:00:24,931 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:24,934 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-25T11:00:24,935 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@436d23f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-25T11:00:24,936 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T11:00:24,938 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,38299,1732532423554, seqNum=-1] 2024-11-25T11:00:24,939 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T11:00:24,947 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40096, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T11:00:24,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6bfc37bad115,39111,1732532423417 2024-11-25T11:00:24,954 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-25T11:00:24,971 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6bfc37bad115,39111,1732532423417 2024-11-25T11:00:24,971 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3676468b 2024-11-25T11:00:24,972 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-25T11:00:24,976 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-25T11:00:24,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39111 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testBlockEvictionOnGracefulStop1732532424953', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-25T11:00:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39111 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 2024-11-25T11:00:24,981 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-25T11:00:24,982 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:24,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39111 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732532424953" procId is: 4 2024-11-25T11:00:24,989 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-25T11:00:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:25,027 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d320d3867213a702dfaf518a131f4cda, NAME => 'testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732532424953', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:25,077 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:25,078 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(1722): Closing d320d3867213a702dfaf518a131f4cda, disabling compactions & flushes 2024-11-25T11:00:25,078 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:25,078 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:25,078 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. after waiting 0 ms 2024-11-25T11:00:25,078 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:25,078 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:25,078 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732532424953-pool-0 {}] regionserver.HRegion(1676): Region close journal for d320d3867213a702dfaf518a131f4cda: Waiting for close lock at 1732532425077Disabling compacts and flushes for region at 1732532425077Disabling writes for close at 1732532425078 (+1 ms)Writing region close event to WAL at 1732532425078Closed at 1732532425078 2024-11-25T11:00:25,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 execute state=CREATE_TABLE_ADD_TO_META 2024-11-25T11:00:25,080 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732532425080"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732532425080"}]},"ts":"1732532425080"} 2024-11-25T11:00:25,085 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-25T11:00:25,086 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-25T11:00:25,086 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732532424953","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532425086"}]},"ts":"1732532425086"} 2024-11-25T11:00:25,089 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732532424953, state=ENABLING in hbase:meta 2024-11-25T11:00:25,090 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {6bfc37bad115=0} racks are {/default-rack=0} 2024-11-25T11:00:25,090 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-25T11:00:25,090 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-25T11:00:25,091 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-25T11:00:25,091 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-25T11:00:25,091 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-25T11:00:25,091 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-25T11:00:25,091 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-25T11:00:25,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN}] 2024-11-25T11:00:25,093 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN 2024-11-25T11:00:25,094 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN; state=OFFLINE, location=6bfc37bad115,38299,1732532423554; forceNewPlan=false, retain=false 2024-11-25T11:00:25,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:25,247 INFO [6bfc37bad115:39111 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-25T11:00:25,247 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d320d3867213a702dfaf518a131f4cda, regionState=OPENING, regionLocation=6bfc37bad115,38299,1732532423554 2024-11-25T11:00:25,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN because future has completed 2024-11-25T11:00:25,251 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d320d3867213a702dfaf518a131f4cda, server=6bfc37bad115,38299,1732532423554}] 2024-11-25T11:00:25,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:25,309 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-25T11:00:25,408 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:25,408 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => d320d3867213a702dfaf518a131f4cda, NAME => 'testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda.', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:25,409 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732532424953 d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,409 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:25,409 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,409 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,411 INFO [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,419 INFO [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d320d3867213a702dfaf518a131f4cda columnFamilyName family 2024-11-25T11:00:25,420 DEBUG [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:25,420 INFO [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] regionserver.HStore(327): Store=d320d3867213a702dfaf518a131f4cda/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:25,420 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,421 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,421 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,421 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,421 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,423 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,436 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-25T11:00:25,437 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened d320d3867213a702dfaf518a131f4cda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68300117, jitterRate=0.017751052975654602}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T11:00:25,437 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:25,439 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for d320d3867213a702dfaf518a131f4cda: Running coprocessor pre-open hook at 1732532425409Writing region info on filesystem at 1732532425409Initializing all the Stores at 1732532425410 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532425410Cleaning up temporary data from old regions at 1732532425421 (+11 ms)Running coprocessor post-open hooks at 1732532425437 (+16 ms)Region opened successfully at 1732532425439 (+2 ms) 2024-11-25T11:00:25,441 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda., pid=6, masterSystemTime=1732532425403 2024-11-25T11:00:25,445 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:25,446 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:25,446 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=d320d3867213a702dfaf518a131f4cda, regionState=OPEN, openSeqNum=2, regionLocation=6bfc37bad115,38299,1732532423554 2024-11-25T11:00:25,449 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure d320d3867213a702dfaf518a131f4cda, server=6bfc37bad115,38299,1732532423554 because future has completed 2024-11-25T11:00:25,454 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39111 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=6bfc37bad115,38299,1732532423554, table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-25T11:00:25,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-25T11:00:25,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure d320d3867213a702dfaf518a131f4cda, server=6bfc37bad115,38299,1732532423554 in 206 msec 2024-11-25T11:00:25,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-25T11:00:25,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN in 371 msec 2024-11-25T11:00:25,466 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-25T11:00:25,466 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732532424953","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732532425466"}]},"ts":"1732532425466"} 2024-11-25T11:00:25,469 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732532424953, state=ENABLED in hbase:meta 2024-11-25T11:00:25,470 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 execute state=CREATE_TABLE_POST_OPERATION 2024-11-25T11:00:25,472 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732532424953 in 492 msec 2024-11-25T11:00:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39111 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-25T11:00:25,615 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732532424953 completed 2024-11-25T11:00:25,615 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732532424953 get assigned. Timeout = 60000ms 2024-11-25T11:00:25,616 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T11:00:25,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732532424953 assigned to meta. Checking AM states. 2024-11-25T11:00:25,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-25T11:00:25,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732532424953 assigned. 2024-11-25T11:00:25,629 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732532424953', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda., hostname=6bfc37bad115,38299,1732532423554, seqNum=2] 2024-11-25T11:00:25,637 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d320d3867213a702dfaf518a131f4cda 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-25T11:00:25,654 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f2275874edac435ea58a98af6fa49a01 2024-11-25T11:00:25,675 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/.tmp/family/f2275874edac435ea58a98af6fa49a01 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-25T11:00:25,676 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/.tmp/family/f2275874edac435ea58a98af6fa49a01 2024-11-25T11:00:25,680 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/.tmp/family/f2275874edac435ea58a98af6fa49a01 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01 2024-11-25T11:00:25,682 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01, delay=909 ms 2024-11-25T11:00:25,683 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01, entries=2, sequenceid=6, filesize=4.9 K 2024-11-25T11:00:25,684 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for d320d3867213a702dfaf518a131f4cda in 47ms, sequenceid=6, compaction requested=false 2024-11-25T11:00:25,685 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732532424953' 2024-11-25T11:00:25,686 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for d320d3867213a702dfaf518a131f4cda: 2024-11-25T11:00:26,592 DEBUG [hfile-prefetch-1732532422734 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f2275874edac435ea58a98af6fa49a01_0. 2024-11-25T11:00:26,593 DEBUG [hfile-prefetch-1732532422734 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key f2275874edac435ea58a98af6fa49a01_117. 2024-11-25T11:00:26,593 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01, with total blocks 2, and data blocks 1 2024-11-25T11:00:26,593 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-25T11:00:26,593 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-25T11:00:26,593 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01. Will sleep 100ms and try the verification again. 2024-11-25T11:00:26,682 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-25T11:00:26,683 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-25T11:00:26,683 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:26,683 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:26,686 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;6bfc37bad115:38299,5,FailOnTimeoutGroup] 2024-11-25T11:00:26,686 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6bfc37bad115,38299,1732532423554' ***** 2024-11-25T11:00:26,687 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-25T11:00:26,687 INFO [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T11:00:26,687 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-25T11:00:26,687 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T11:00:26,687 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T11:00:26,687 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732532426683 2024-11-25T11:00:26,687 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-25T11:00:26,687 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T11:00:26,687 INFO [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence 2024-11-25T11:00:26,687 INFO [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T11:00:26,687 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T11:00:26,694 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01, with total blocks 2, and data blocks 1 2024-11-25T11:00:26,694 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-25T11:00:26,694 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2293): found block for file f2275874edac435ea58a98af6fa49a01 in the backing map. Acquiring read lock for offset 0 2024-11-25T11:00:26,694 DEBUG [hfile-prefetch-1732532422734 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01 has now been fully cached. 2024-11-25T11:00:26,694 DEBUG [hfile-prefetch-1732532422734 {}] hfile.PrefetchExecutor(121): Prefetch completed for f2275874edac435ea58a98af6fa49a01 2024-11-25T11:00:26,698 DEBUG [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1487): Thread RS:0;6bfc37bad115:38299 started persisting bucket cache to file 2024-11-25T11:00:26,698 DEBUG [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:26,698 DEBUG [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-25T11:00:26,702 DEBUG [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-25T11:00:26,702 DEBUG [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732532426698 2024-11-25T11:00:26,702 DEBUG [RS:0;6bfc37bad115:38299 {}] bucket.BucketCache(1509): Thread RS:0;6bfc37bad115:38299 finished persisting bucket cache to file, renaming 2024-11-25T11:00:26,703 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T11:00:26,703 INFO [RS:0;6bfc37bad115:38299 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T11:00:26,703 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T11:00:26,703 INFO [RS:0;6bfc37bad115:38299 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T11:00:26,703 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(3091): Received CLOSE for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:26,703 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(959): stopping server 6bfc37bad115,38299,1732532423554 2024-11-25T11:00:26,704 INFO [RS:0;6bfc37bad115:38299 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T11:00:26,704 INFO [RS:0;6bfc37bad115:38299 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6bfc37bad115:38299. 2024-11-25T11:00:26,704 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T11:00:26,704 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d320d3867213a702dfaf518a131f4cda, disabling compactions & flushes 2024-11-25T11:00:26,704 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:26,704 DEBUG [RS:0;6bfc37bad115:38299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:26,704 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:26,704 DEBUG [RS:0;6bfc37bad115:38299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:26,704 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. after waiting 0 ms 2024-11-25T11:00:26,704 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:26,704 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T11:00:26,704 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T11:00:26,704 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T11:00:26,704 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T11:00:26,705 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T11:00:26,705 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1325): Online Regions={d320d3867213a702dfaf518a131f4cda=testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda., 1588230740=hbase:meta,,1.1588230740} 2024-11-25T11:00:26,705 DEBUG [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:26,705 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T11:00:26,705 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T11:00:26,705 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T11:00:26,705 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T11:00:26,705 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T11:00:26,705 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T11:00:26,705 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-25T11:00:26,711 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-25T11:00:26,712 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:26,712 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d320d3867213a702dfaf518a131f4cda: Waiting for close lock at 1732532426704Running coprocessor pre-close hooks at 1732532426704Disabling compacts and flushes for region at 1732532426704Disabling writes for close at 1732532426704Writing region close event to WAL at 1732532426705 (+1 ms)Running coprocessor post-close hooks at 1732532426711 (+6 ms)Closed at 1732532426711 2024-11-25T11:00:26,712 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:26,714 INFO [regionserver/6bfc37bad115:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:26,717 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d2df73adf2804683824b5f7945744084 2024-11-25T11:00:26,722 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-25T11:00:26,723 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-25T11:00:26,725 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T11:00:26,725 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-25T11:00:26,726 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-25T11:00:26,726 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-25T11:00:26,738 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/d2df73adf2804683824b5f7945744084 is 209, key is testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda./info:regioninfo/1732532425446/Put/seqid=0 2024-11-25T11:00:26,739 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/d2df73adf2804683824b5f7945744084 2024-11-25T11:00:26,751 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a5c2a22053784ab69d1ff8b02c2baff4 2024-11-25T11:00:26,770 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/ns/a5c2a22053784ab69d1ff8b02c2baff4 is 43, key is default/ns:d/1732532424828/Put/seqid=0 2024-11-25T11:00:26,771 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/ns/a5c2a22053784ab69d1ff8b02c2baff4 2024-11-25T11:00:26,782 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1fe15bdbc5c04902b086761f20629f98 2024-11-25T11:00:26,796 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/table/1fe15bdbc5c04902b086761f20629f98 is 80, key is testBlockEvictionOnGracefulStop1732532424953/table:state/1732532425466/Put/seqid=0 2024-11-25T11:00:26,797 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/table/1fe15bdbc5c04902b086761f20629f98 2024-11-25T11:00:26,800 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/d2df73adf2804683824b5f7945744084 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/d2df73adf2804683824b5f7945744084 2024-11-25T11:00:26,802 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/d2df73adf2804683824b5f7945744084, delay=986 ms 2024-11-25T11:00:26,803 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/d2df73adf2804683824b5f7945744084, entries=10, sequenceid=11, filesize=7.0 K 2024-11-25T11:00:26,803 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/ns/a5c2a22053784ab69d1ff8b02c2baff4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/a5c2a22053784ab69d1ff8b02c2baff4 2024-11-25T11:00:26,805 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/a5c2a22053784ab69d1ff8b02c2baff4, delay=919 ms 2024-11-25T11:00:26,806 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/a5c2a22053784ab69d1ff8b02c2baff4, entries=2, sequenceid=11, filesize=5.0 K 2024-11-25T11:00:26,806 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/table/1fe15bdbc5c04902b086761f20629f98 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/1fe15bdbc5c04902b086761f20629f98 2024-11-25T11:00:26,807 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/1fe15bdbc5c04902b086761f20629f98, delay=981 ms 2024-11-25T11:00:26,808 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/1fe15bdbc5c04902b086761f20629f98, entries=2, sequenceid=11, filesize=5.3 K 2024-11-25T11:00:26,809 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false 2024-11-25T11:00:26,810 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/d2df73adf2804683824b5f7945744084 2024-11-25T11:00:26,811 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/a5c2a22053784ab69d1ff8b02c2baff4 2024-11-25T11:00:26,811 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/1fe15bdbc5c04902b086761f20629f98 2024-11-25T11:00:26,817 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-25T11:00:26,817 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T11:00:26,817 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:26,817 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732532426705Running coprocessor pre-close hooks at 1732532426705Disabling compacts and flushes for region at 1732532426705Disabling writes for close at 1732532426705Obtaining lock to block concurrent updates at 1732532426705Preparing flush snapshotting stores in 1588230740 at 1732532426706 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1732532426706Flushing stores of hbase:meta,,1.1588230740 at 1732532426706Flushing 1588230740/info: creating writer at 1732532426706Flushing 1588230740/info: appending metadata at 1732532426737 (+31 ms)Flushing 1588230740/info: closing flushed file at 1732532426738 (+1 ms)Flushing 1588230740/ns: creating writer at 1732532426742 (+4 ms)Flushing 1588230740/ns: appending metadata at 1732532426769 (+27 ms)Flushing 1588230740/ns: closing flushed file at 1732532426769Flushing 1588230740/table: creating writer at 1732532426774 (+5 ms)Flushing 1588230740/table: appending metadata at 1732532426796 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732532426796Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c8eccb9: reopening flushed file at 1732532426800 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5443159a: reopening flushed file at 1732532426803 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@335e3c4a: reopening flushed file at 1732532426806 (+3 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 104ms, sequenceid=11, compaction requested=false at 1732532426809 (+3 ms)Writing region close event to WAL at 1732532426812 (+3 ms)Running coprocessor post-close hooks at 1732532426817 (+5 ms)Closed at 1732532426817 2024-11-25T11:00:26,818 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:26,905 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(976): stopping server 6bfc37bad115,38299,1732532423554; all regions closed. 2024-11-25T11:00:26,908 DEBUG [RS:0;6bfc37bad115:38299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:26,908 INFO [RS:0;6bfc37bad115:38299 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C38299%2C1732532423554.meta:.meta(num 1732532424729) 2024-11-25T11:00:26,910 DEBUG [RS:0;6bfc37bad115:38299 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:26,910 INFO [RS:0;6bfc37bad115:38299 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C38299%2C1732532423554:(num 1732532424369) 2024-11-25T11:00:26,910 DEBUG [RS:0;6bfc37bad115:38299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:26,910 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:26,910 INFO [RS:0;6bfc37bad115:38299 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T11:00:26,910 INFO [RS:0;6bfc37bad115:38299 {}] hbase.ChoreService(370): Chore service for: regionserver/6bfc37bad115:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T11:00:26,911 INFO [RS:0;6bfc37bad115:38299 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T11:00:26,911 INFO [regionserver/6bfc37bad115:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T11:00:26,911 INFO [RS:0;6bfc37bad115:38299 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38299 2024-11-25T11:00:26,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:26,916 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6bfc37bad115,38299,1732532423554 2024-11-25T11:00:26,917 INFO [RS:0;6bfc37bad115:38299 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T11:00:26,917 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6bfc37bad115,38299,1732532423554] 2024-11-25T11:00:26,919 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6bfc37bad115,38299,1732532423554 already deleted, retry=false 2024-11-25T11:00:26,919 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6bfc37bad115,38299,1732532423554 on 6bfc37bad115,39111,1732532423417 2024-11-25T11:00:26,924 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6bfc37bad115,38299,1732532423554, splitWal=true, meta=true 2024-11-25T11:00:26,927 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 6bfc37bad115,38299,1732532423554 (carryingMeta=true) 6bfc37bad115,38299,1732532423554/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2e9d2087[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-25T11:00:26,927 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6bfc37bad115,38299,1732532423554, splitWal=true, meta=true 2024-11-25T11:00:26,928 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6bfc37bad115,38299,1732532423554, splitWal=true, meta=true, isMeta: true 2024-11-25T11:00:26,929 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554-splitting 2024-11-25T11:00:26,929 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554-splitting dir is empty, no logs to split. 2024-11-25T11:00:26,929 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6bfc37bad115,38299,1732532423554 WAL count=0, meta=true 2024-11-25T11:00:26,932 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554-splitting dir is empty, no logs to split. 2024-11-25T11:00:26,932 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6bfc37bad115,38299,1732532423554 WAL count=0, meta=true 2024-11-25T11:00:26,932 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6bfc37bad115,38299,1732532423554 WAL splitting is done? wals=0, meta=true 2024-11-25T11:00:26,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-25T11:00:26,935 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T11:00:26,936 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T11:00:27,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:27,019 INFO [RS:0;6bfc37bad115:38299 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T11:00:27,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38299-0x1007698dd510001, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:27,019 INFO [RS:0;6bfc37bad115:38299 {}] regionserver.HRegionServer(1031): Exiting; stopping=6bfc37bad115,38299,1732532423554; zookeeper connection closed. 2024-11-25T11:00:27,019 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b3761e6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b3761e6 2024-11-25T11:00:27,086 DEBUG [6bfc37bad115:39111 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-25T11:00:27,087 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6bfc37bad115,36685,1732532423630 2024-11-25T11:00:27,088 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,36685,1732532423630, state=OPENING 2024-11-25T11:00:27,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:27,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:27,090 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-25T11:00:27,090 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:27,090 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:27,090 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,36685,1732532423630}] 2024-11-25T11:00:27,211 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6bfc37bad115:0 server-side Connection retries=45 2024-11-25T11:00:27,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:27,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:27,211 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-25T11:00:27,211 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-25T11:00:27,212 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-25T11:00:27,212 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-25T11:00:27,212 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-25T11:00:27,213 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35003 2024-11-25T11:00:27,215 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35003 connecting to ZooKeeper ensemble=127.0.0.1:61363 2024-11-25T11:00:27,215 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:27,216 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:27,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:350030x0, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-25T11:00:27,220 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:350030x0, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-25T11:00:27,220 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-25T11:00:27,221 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35003-0x1007698dd510003 connected 2024-11-25T11:00:27,221 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-25T11:00:27,226 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.cache 2024-11-25T11:00:27,226 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-25T11:00:27,226 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-25T11:00:27,226 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-25T11:00:27,227 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-25T11:00:27,228 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-25T11:00:27,229 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-25T11:00:27,229 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-25T11:00:27,230 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-25T11:00:27,231 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-11-25T11:00:27,233 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-25T11:00:27,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35003 2024-11-25T11:00:27,238 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35003 2024-11-25T11:00:27,241 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35003 2024-11-25T11:00:27,242 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-25T11:00:27,242 DEBUG [Thread-208 {}] bucket.BucketCache(1584): Cached File Entry:> 2024-11-25T11:00:27,242 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35003 2024-11-25T11:00:27,243 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-25T11:00:27,243 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-11-25T11:00:27,243 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence is complete. 2024-11-25T11:00:27,244 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35003 2024-11-25T11:00:27,245 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-25T11:00:27,247 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40205, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-25T11:00:27,250 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-25T11:00:27,250 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:27,251 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(746): ClusterId : ba7d93d3-e485-4440-a168-037d33c3e62b 2024-11-25T11:00:27,251 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-25T11:00:27,251 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-25T11:00:27,252 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C36685%2C1732532423630.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36685,1732532423630, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:27,257 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-25T11:00:27,257 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-25T11:00:27,261 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-25T11:00:27,262 DEBUG [RS:2;6bfc37bad115:35003 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@215ee98d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6bfc37bad115/172.17.0.3:0 2024-11-25T11:00:27,281 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,36685,1732532423630/6bfc37bad115%2C36685%2C1732532423630.meta.1732532427253.meta 2024-11-25T11:00:27,281 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:27,282 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:27,282 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-25T11:00:27,282 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-25T11:00:27,283 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-25T11:00:27,283 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-25T11:00:27,283 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:27,283 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-25T11:00:27,283 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-25T11:00:27,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-25T11:00:27,285 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-25T11:00:27,285 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:27,287 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/d2df73adf2804683824b5f7945744084, delay=902 ms 2024-11-25T11:00:27,289 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/d2df73adf2804683824b5f7945744084 2024-11-25T11:00:27,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:27,289 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-25T11:00:27,289 DEBUG [RS:2;6bfc37bad115:35003 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;6bfc37bad115:35003 2024-11-25T11:00:27,290 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-25T11:00:27,290 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-25T11:00:27,290 DEBUG [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-25T11:00:27,290 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-25T11:00:27,290 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:27,291 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(2659): reportForDuty to master=6bfc37bad115,39111,1732532423417 with port=35003, startcode=1732532427210 2024-11-25T11:00:27,291 DEBUG [RS:2;6bfc37bad115:35003 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-25T11:00:27,294 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/a5c2a22053784ab69d1ff8b02c2baff4, delay=934 ms 2024-11-25T11:00:27,295 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/a5c2a22053784ab69d1ff8b02c2baff4 2024-11-25T11:00:27,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:27,296 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-25T11:00:27,296 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-25T11:00:27,297 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:27,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:27,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-25T11:00:27,298 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-25T11:00:27,298 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:27,311 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42449, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-25T11:00:27,312 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39111 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,312 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39111 {}] master.ServerManager(517): Registering regionserver=6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,312 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/1fe15bdbc5c04902b086761f20629f98, delay=994 ms 2024-11-25T11:00:27,313 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/1fe15bdbc5c04902b086761f20629f98 2024-11-25T11:00:27,314 DEBUG [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d 2024-11-25T11:00:27,314 DEBUG [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-25T11:00:27,314 DEBUG [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-25T11:00:27,314 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-25T11:00:27,314 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-25T11:00:27,315 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:27,316 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740 2024-11-25T11:00:27,316 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-25T11:00:27,316 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-25T11:00:27,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:27,317 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-25T11:00:27,317 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6bfc37bad115,35003,1732532427210] 2024-11-25T11:00:27,318 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-25T11:00:27,319 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74439786, jitterRate=0.10923925042152405}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-25T11:00:27,319 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-25T11:00:27,319 DEBUG [RS:2;6bfc37bad115:35003 {}] zookeeper.ZKUtil(111): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,319 WARN [RS:2;6bfc37bad115:35003 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-25T11:00:27,320 INFO [RS:2;6bfc37bad115:35003 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-25T11:00:27,320 DEBUG [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,320 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732532427283Writing region info on filesystem at 1732532427283Initializing all the Stores at 1732532427284 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532427284Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532427284Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532427284Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732532427284Cleaning up temporary data from old regions at 1732532427316 (+32 ms)Running coprocessor post-open hooks at 1732532427319 (+3 ms)Region opened successfully at 1732532427320 (+1 ms) 2024-11-25T11:00:27,321 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1732532427245 2024-11-25T11:00:27,326 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=6bfc37bad115,36685,1732532423630 2024-11-25T11:00:27,327 DEBUG [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-25T11:00:27,327 INFO [RS_OPEN_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-25T11:00:27,328 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6bfc37bad115,36685,1732532423630, state=OPEN 2024-11-25T11:00:27,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:27,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:27,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-25T11:00:27,330 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:27,330 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:27,330 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6bfc37bad115,36685,1732532423630 2024-11-25T11:00:27,330 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-25T11:00:27,330 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-25T11:00:27,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-25T11:00:27,333 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6bfc37bad115,36685,1732532423630 in 240 msec 2024-11-25T11:00:27,334 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-25T11:00:27,334 INFO [RS:2;6bfc37bad115:35003 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-25T11:00:27,335 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,335 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-25T11:00:27,335 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-25T11:00:27,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-25T11:00:27,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 399 msec 2024-11-25T11:00:27,336 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,336 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,336 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,336 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,336 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,336 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,336 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 6bfc37bad115,38299,1732532423554 had 2 regions 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6bfc37bad115:0, corePoolSize=2, maxPoolSize=2 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6bfc37bad115:0, corePoolSize=1, maxPoolSize=1 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:27,337 DEBUG [RS:2;6bfc37bad115:35003 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6bfc37bad115:0, corePoolSize=3, maxPoolSize=3 2024-11-25T11:00:27,337 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6bfc37bad115,38299,1732532423554, splitWal=true, meta=true, isMeta: false 2024-11-25T11:00:27,338 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554-splitting dir is empty, no logs to split. 2024-11-25T11:00:27,338 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6bfc37bad115,38299,1732532423554 WAL count=0, meta=false 2024-11-25T11:00:27,340 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,38299,1732532423554-splitting dir is empty, no logs to split. 2024-11-25T11:00:27,340 INFO [PEWorker-1 {}] master.SplitWALManager(105): 6bfc37bad115,38299,1732532423554 WAL count=0, meta=false 2024-11-25T11:00:27,340 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 6bfc37bad115,38299,1732532423554 WAL splitting is done? wals=0, meta=false 2024-11-25T11:00:27,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN}] 2024-11-25T11:00:27,343 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN 2024-11-25T11:00:27,344 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-25T11:00:27,344 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,344 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,345 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,345 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,345 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,345 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,35003,1732532427210-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-25T11:00:27,368 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-25T11:00:27,368 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=6bfc37bad115,35003,1732532427210-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,369 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,369 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.Replication(171): 6bfc37bad115,35003,1732532427210 started 2024-11-25T11:00:27,391 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-25T11:00:27,392 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(1482): Serving as 6bfc37bad115,35003,1732532427210, RpcServer on 6bfc37bad115/172.17.0.3:35003, sessionid=0x1007698dd510003 2024-11-25T11:00:27,392 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-25T11:00:27,392 DEBUG [RS:2;6bfc37bad115:35003 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,392 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,35003,1732532427210' 2024-11-25T11:00:27,392 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-25T11:00:27,393 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-25T11:00:27,395 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-25T11:00:27,395 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-25T11:00:27,395 DEBUG [RS:2;6bfc37bad115:35003 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,395 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6bfc37bad115,35003,1732532427210' 2024-11-25T11:00:27,395 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-25T11:00:27,396 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-25T11:00:27,397 DEBUG [RS:2;6bfc37bad115:35003 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-25T11:00:27,397 INFO [RS:2;6bfc37bad115:35003 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-25T11:00:27,397 INFO [RS:2;6bfc37bad115:35003 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-25T11:00:27,495 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d320d3867213a702dfaf518a131f4cda, regionState=OPENING, regionLocation=6bfc37bad115,36685,1732532423630 2024-11-25T11:00:27,497 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6bfc37bad115:38299 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6bfc37bad115/172.17.0.3:38299 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-25T11:00:27,498 INFO [RS:2;6bfc37bad115:35003 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-25T11:00:27,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,38299,1732532423554, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,38299,1732532423554, seqNum=-1, error=java.net.ConnectException: Call to address=6bfc37bad115:38299 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6bfc37bad115/172.17.0.3:38299 2024-11-25T11:00:27,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,38299,1732532423554, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-25T11:00:27,500 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,38299,1732532423554, seqNum=-1 from cache 2024-11-25T11:00:27,501 INFO [RS:2;6bfc37bad115:35003 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6bfc37bad115%2C35003%2C1732532427210, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,35003,1732532427210, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs, maxLogs=55 2024-11-25T11:00:27,503 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.FailedServers(52): Added failed server with address 6bfc37bad115:38299 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6bfc37bad115/172.17.0.3:38299 2024-11-25T11:00:27,528 INFO [RS:2;6bfc37bad115:35003 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/WALs/6bfc37bad115,35003,1732532427210/6bfc37bad115%2C35003%2C1732532427210.1732532427502 2024-11-25T11:00:27,529 DEBUG [RS:2;6bfc37bad115:35003 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-25T11:00:27,603 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-25T11:00:27,604 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6bfc37bad115,36685,1732532423630, seqNum=-1] 2024-11-25T11:00:27,604 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-25T11:00:27,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58039, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-25T11:00:27,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN because future has completed 2024-11-25T11:00:27,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d320d3867213a702dfaf518a131f4cda, server=6bfc37bad115,36685,1732532423630}] 2024-11-25T11:00:27,766 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,766 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => d320d3867213a702dfaf518a131f4cda, NAME => 'testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda.', STARTKEY => '', ENDKEY => ''} 2024-11-25T11:00:27,767 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732532424953 d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,767 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-25T11:00:27,767 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,767 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,768 INFO [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,769 INFO [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d320d3867213a702dfaf518a131f4cda columnFamilyName family 2024-11-25T11:00:27,769 DEBUG [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-25T11:00:27,771 DEBUG [StoreFileOpener-d320d3867213a702dfaf518a131f4cda-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01, delay=932 ms 2024-11-25T11:00:27,772 DEBUG [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01 2024-11-25T11:00:27,772 INFO [StoreOpener-d320d3867213a702dfaf518a131f4cda-1 {}] regionserver.HStore(327): Store=d320d3867213a702dfaf518a131f4cda/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-25T11:00:27,773 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,773 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,774 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,774 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,774 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,775 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,775 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened d320d3867213a702dfaf518a131f4cda; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61503393, jitterRate=-0.08352802693843842}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-25T11:00:27,776 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,777 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for d320d3867213a702dfaf518a131f4cda: Running coprocessor pre-open hook at 1732532427767Writing region info on filesystem at 1732532427767Initializing all the Stores at 1732532427767Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732532427767Cleaning up temporary data from old regions at 1732532427774 (+7 ms)Running coprocessor post-open hooks at 1732532427776 (+2 ms)Region opened successfully at 1732532427777 (+1 ms) 2024-11-25T11:00:27,777 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda., pid=11, masterSystemTime=1732532427763 2024-11-25T11:00:27,781 DEBUG [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,781 INFO [RS_OPEN_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,781 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=d320d3867213a702dfaf518a131f4cda, regionState=OPEN, openSeqNum=10, regionLocation=6bfc37bad115,36685,1732532423630 2024-11-25T11:00:27,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure d320d3867213a702dfaf518a131f4cda, server=6bfc37bad115,36685,1732532423630 because future has completed 2024-11-25T11:00:27,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-25T11:00:27,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure d320d3867213a702dfaf518a131f4cda, server=6bfc37bad115,36685,1732532423630 in 175 msec 2024-11-25T11:00:27,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-25T11:00:27,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732532424953, region=d320d3867213a702dfaf518a131f4cda, ASSIGN in 445 msec 2024-11-25T11:00:27,789 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(291): removed crashed server 6bfc37bad115,38299,1732532423554 after splitting done 2024-11-25T11:00:27,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 6bfc37bad115,38299,1732532423554, splitWal=true, meta=true in 870 msec 2024-11-25T11:00:27,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-25T11:00:27,892 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T11:00:27,893 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:27,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:27,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:27,893 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-25T11:00:27,893 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-25T11:00:27,893 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=874745851, stopped=false 2024-11-25T11:00:27,893 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6bfc37bad115,39111,1732532423417 2024-11-25T11:00:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:27,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-25T11:00:27,896 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:27,896 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T11:00:27,896 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-25T11:00:27,896 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:27,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:27,896 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6bfc37bad115,36685,1732532423630' ***** 2024-11-25T11:00:27,897 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T11:00:27,897 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6bfc37bad115,35003,1732532427210' ***** 2024-11-25T11:00:27,897 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-25T11:00:27,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:27,898 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:27,899 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-25T11:00:27,900 INFO [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T11:00:27,900 INFO [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence 2024-11-25T11:00:27,900 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T11:00:27,901 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T11:00:27,901 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T11:00:27,901 INFO [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T11:00:27,902 INFO [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1738): Disabling cache 2024-11-25T11:00:27,902 INFO [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/72902a18-dffb-392b-5303-ad11193f8a7d/bucket.persistence 2024-11-25T11:00:27,902 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T11:00:27,902 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-25T11:00:27,902 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-25T11:00:27,904 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-25T11:00:27,904 INFO [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-25T11:00:27,905 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-25T11:00:27,912 DEBUG [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1487): Thread RS:2;6bfc37bad115:35003 started persisting bucket cache to file 2024-11-25T11:00:27,912 DEBUG [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:27,912 DEBUG [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-11-25T11:00:27,915 DEBUG [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1487): Thread RS:1;6bfc37bad115:36685 started persisting bucket cache to file 2024-11-25T11:00:27,915 DEBUG [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-25T11:00:27,915 DEBUG [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-25T11:00:27,916 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T11:00:27,919 DEBUG [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-25T11:00:27,919 DEBUG [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-25T11:00:27,919 DEBUG [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1732532427912 2024-11-25T11:00:27,919 DEBUG [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732532427915 2024-11-25T11:00:27,919 DEBUG [RS:2;6bfc37bad115:35003 {}] bucket.BucketCache(1509): Thread RS:2;6bfc37bad115:35003 finished persisting bucket cache to file, renaming 2024-11-25T11:00:27,919 DEBUG [RS:1;6bfc37bad115:36685 {}] bucket.BucketCache(1509): Thread RS:1;6bfc37bad115:36685 finished persisting bucket cache to file, renaming 2024-11-25T11:00:27,919 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T11:00:27,920 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-25T11:00:27,920 INFO [RS:2;6bfc37bad115:35003 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T11:00:27,920 INFO [RS:2;6bfc37bad115:35003 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T11:00:27,920 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(959): stopping server 6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,920 INFO [RS:2;6bfc37bad115:35003 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T11:00:27,920 INFO [RS:2;6bfc37bad115:35003 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;6bfc37bad115:35003. 2024-11-25T11:00:27,920 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-25T11:00:27,920 DEBUG [RS:2;6bfc37bad115:35003 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:27,920 INFO [RS:1;6bfc37bad115:36685 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-25T11:00:27,920 DEBUG [RS:2;6bfc37bad115:35003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:27,920 INFO [RS:1;6bfc37bad115:36685 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-25T11:00:27,920 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(3091): Received CLOSE for d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,921 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(976): stopping server 6bfc37bad115,35003,1732532427210; all regions closed. 2024-11-25T11:00:27,921 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(959): stopping server 6bfc37bad115,36685,1732532423630 2024-11-25T11:00:27,921 INFO [RS:1;6bfc37bad115:36685 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T11:00:27,921 INFO [RS:1;6bfc37bad115:36685 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6bfc37bad115:36685. 2024-11-25T11:00:27,921 DEBUG [RS:1;6bfc37bad115:36685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-25T11:00:27,921 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T11:00:27,921 DEBUG [RS:1;6bfc37bad115:36685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:27,921 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d320d3867213a702dfaf518a131f4cda, disabling compactions & flushes 2024-11-25T11:00:27,921 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,921 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,921 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T11:00:27,921 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. after waiting 0 ms 2024-11-25T11:00:27,921 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T11:00:27,921 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,921 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T11:00:27,921 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-25T11:00:27,924 INFO [regionserver/6bfc37bad115:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:27,924 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-25T11:00:27,925 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1325): Online Regions={d320d3867213a702dfaf518a131f4cda=testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda., 1588230740=hbase:meta,,1.1588230740} 2024-11-25T11:00:27,925 DEBUG [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, d320d3867213a702dfaf518a131f4cda 2024-11-25T11:00:27,925 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-25T11:00:27,925 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-25T11:00:27,925 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-25T11:00:27,925 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-25T11:00:27,925 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-25T11:00:27,925 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-25T11:00:27,926 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-25T11:00:27,926 DEBUG [StoreFileCloser-d320d3867213a702dfaf518a131f4cda-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/family/f2275874edac435ea58a98af6fa49a01 2024-11-25T11:00:27,929 DEBUG [RS:2;6bfc37bad115:35003 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:27,929 INFO [RS:2;6bfc37bad115:35003 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C35003%2C1732532427210:(num 1732532427502) 2024-11-25T11:00:27,929 DEBUG [RS:2;6bfc37bad115:35003 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:27,929 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:27,929 INFO [RS:2;6bfc37bad115:35003 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T11:00:27,929 INFO [RS:2;6bfc37bad115:35003 {}] hbase.ChoreService(370): Chore service for: regionserver/6bfc37bad115:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-25T11:00:27,929 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-25T11:00:27,929 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-25T11:00:27,929 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-25T11:00:27,930 INFO [RS:2;6bfc37bad115:35003 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T11:00:27,930 INFO [RS:2;6bfc37bad115:35003 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35003 2024-11-25T11:00:27,933 INFO [regionserver/6bfc37bad115:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T11:00:27,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6bfc37bad115,35003,1732532427210 2024-11-25T11:00:27,934 INFO [RS:2;6bfc37bad115:35003 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T11:00:27,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:27,936 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6bfc37bad115,35003,1732532427210] 2024-11-25T11:00:27,938 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6bfc37bad115,35003,1732532427210 already deleted, retry=false 2024-11-25T11:00:27,938 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6bfc37bad115,35003,1732532427210 expired; onlineServers=1 2024-11-25T11:00:27,941 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/default/testBlockEvictionOnGracefulStop1732532424953/d320d3867213a702dfaf518a131f4cda/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-25T11:00:27,941 INFO [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,942 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d320d3867213a702dfaf518a131f4cda: Waiting for close lock at 1732532427921Running coprocessor pre-close hooks at 1732532427921Disabling compacts and flushes for region at 1732532427921Disabling writes for close at 1732532427921Writing region close event to WAL at 1732532427927 (+6 ms)Running coprocessor post-close hooks at 1732532427941 (+14 ms)Closed at 1732532427941 2024-11-25T11:00:27,942 DEBUG [RS_CLOSE_REGION-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda. 2024-11-25T11:00:27,949 INFO [regionserver/6bfc37bad115:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:27,951 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on eb4c5fc6ae2c45b2ace36261d141eae6 2024-11-25T11:00:27,966 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/eb4c5fc6ae2c45b2ace36261d141eae6 is 209, key is testBlockEvictionOnGracefulStop1732532424953,,1732532424978.d320d3867213a702dfaf518a131f4cda./info:regioninfo/1732532427781/Put/seqid=0 2024-11-25T11:00:27,967 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/eb4c5fc6ae2c45b2ace36261d141eae6 2024-11-25T11:00:27,970 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/.tmp/info/eb4c5fc6ae2c45b2ace36261d141eae6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/eb4c5fc6ae2c45b2ace36261d141eae6 2024-11-25T11:00:27,971 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/eb4c5fc6ae2c45b2ace36261d141eae6, delay=998 ms 2024-11-25T11:00:27,972 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/eb4c5fc6ae2c45b2ace36261d141eae6, entries=8, sequenceid=19, filesize=6.6 K 2024-11-25T11:00:27,973 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 48ms, sequenceid=19, compaction requested=false 2024-11-25T11:00:27,974 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/d2df73adf2804683824b5f7945744084 2024-11-25T11:00:27,974 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/info/eb4c5fc6ae2c45b2ace36261d141eae6 2024-11-25T11:00:27,976 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/ns/a5c2a22053784ab69d1ff8b02c2baff4 2024-11-25T11:00:27,978 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/table/1fe15bdbc5c04902b086761f20629f98 2024-11-25T11:00:27,985 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-25T11:00:27,985 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-25T11:00:27,985 INFO [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:27,986 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732532427925Running coprocessor pre-close hooks at 1732532427925Disabling compacts and flushes for region at 1732532427925Disabling writes for close at 1732532427925Obtaining lock to block concurrent updates at 1732532427926 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732532427926Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1732532427926Flushing stores of hbase:meta,,1.1588230740 at 1732532427926Flushing 1588230740/info: creating writer at 1732532427927 (+1 ms)Flushing 1588230740/info: appending metadata at 1732532427966 (+39 ms)Flushing 1588230740/info: closing flushed file at 1732532427966Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@401c1cb7: reopening flushed file at 1732532427970 (+4 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 48ms, sequenceid=19, compaction requested=false at 1732532427973 (+3 ms)Writing region close event to WAL at 1732532427979 (+6 ms)Running coprocessor post-close hooks at 1732532427985 (+6 ms)Closed at 1732532427985 2024-11-25T11:00:27,986 DEBUG [RS_CLOSE_META-regionserver/6bfc37bad115:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-25T11:00:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:28,037 INFO [RS:2;6bfc37bad115:35003 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T11:00:28,037 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35003-0x1007698dd510003, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:28,037 INFO [RS:2;6bfc37bad115:35003 {}] regionserver.HRegionServer(1031): Exiting; stopping=6bfc37bad115,35003,1732532427210; zookeeper connection closed. 2024-11-25T11:00:28,037 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@206275fc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@206275fc 2024-11-25T11:00:28,125 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(976): stopping server 6bfc37bad115,36685,1732532423630; all regions closed. 2024-11-25T11:00:28,128 DEBUG [RS:1;6bfc37bad115:36685 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:28,128 INFO [RS:1;6bfc37bad115:36685 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C36685%2C1732532423630.meta:.meta(num 1732532427253) 2024-11-25T11:00:28,130 DEBUG [RS:1;6bfc37bad115:36685 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/oldWALs 2024-11-25T11:00:28,130 INFO [RS:1;6bfc37bad115:36685 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6bfc37bad115%2C36685%2C1732532423630:(num 1732532424377) 2024-11-25T11:00:28,130 DEBUG [RS:1;6bfc37bad115:36685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-25T11:00:28,130 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.LeaseManager(133): Closed leases 2024-11-25T11:00:28,135 INFO [RS:1;6bfc37bad115:36685 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T11:00:28,136 INFO [RS:1;6bfc37bad115:36685 {}] hbase.ChoreService(370): Chore service for: regionserver/6bfc37bad115:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-25T11:00:28,136 INFO [RS:1;6bfc37bad115:36685 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T11:00:28,136 INFO [regionserver/6bfc37bad115:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T11:00:28,136 INFO [RS:1;6bfc37bad115:36685 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36685 2024-11-25T11:00:28,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-25T11:00:28,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6bfc37bad115,36685,1732532423630 2024-11-25T11:00:28,140 INFO [RS:1;6bfc37bad115:36685 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T11:00:28,142 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6bfc37bad115,36685,1732532423630] 2024-11-25T11:00:28,143 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6bfc37bad115,36685,1732532423630 already deleted, retry=false 2024-11-25T11:00:28,143 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6bfc37bad115,36685,1732532423630 expired; onlineServers=0 2024-11-25T11:00:28,144 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6bfc37bad115,39111,1732532423417' ***** 2024-11-25T11:00:28,144 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-25T11:00:28,144 INFO [M:0;6bfc37bad115:39111 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-25T11:00:28,144 INFO [M:0;6bfc37bad115:39111 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-25T11:00:28,144 DEBUG [M:0;6bfc37bad115:39111 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-25T11:00:28,144 DEBUG [M:0;6bfc37bad115:39111 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-25T11:00:28,144 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-25T11:00:28,144 DEBUG [master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532424058 {}] cleaner.HFileCleaner(306): Exit Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.large.0-1732532424058,5,FailOnTimeoutGroup] 2024-11-25T11:00:28,144 DEBUG [master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532424061 {}] cleaner.HFileCleaner(306): Exit Thread[master/6bfc37bad115:0:becomeActiveMaster-HFileCleaner.small.0-1732532424061,5,FailOnTimeoutGroup] 2024-11-25T11:00:28,144 INFO [M:0;6bfc37bad115:39111 {}] hbase.ChoreService(370): Chore service for: master/6bfc37bad115:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-25T11:00:28,144 INFO [M:0;6bfc37bad115:39111 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-25T11:00:28,145 DEBUG [M:0;6bfc37bad115:39111 {}] master.HMaster(1795): Stopping service threads 2024-11-25T11:00:28,145 INFO [M:0;6bfc37bad115:39111 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-25T11:00:28,145 INFO [M:0;6bfc37bad115:39111 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-25T11:00:28,145 INFO [M:0;6bfc37bad115:39111 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-25T11:00:28,146 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-25T11:00:28,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-25T11:00:28,149 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-25T11:00:28,149 DEBUG [M:0;6bfc37bad115:39111 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/master already deleted, retry=false 2024-11-25T11:00:28,149 DEBUG [M:0;6bfc37bad115:39111 {}] master.ActiveMasterManager(353): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-11-25T11:00:28,149 INFO [M:0;6bfc37bad115:39111 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/.lastflushedseqids 2024-11-25T11:00:28,158 INFO [M:0;6bfc37bad115:39111 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-25T11:00:28,159 INFO [M:0;6bfc37bad115:39111 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-25T11:00:28,159 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-25T11:00:28,159 INFO [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:28,159 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:28,159 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-25T11:00:28,159 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:28,159 INFO [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.50 KB heapSize=49.91 KB 2024-11-25T11:00:28,170 DEBUG [M:0;6bfc37bad115:39111 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1df1a5f7727e41ec90d715d8d443e585 2024-11-25T11:00:28,191 DEBUG [M:0;6bfc37bad115:39111 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1df1a5f7727e41ec90d715d8d443e585 is 82, key is hbase:meta,,1/info:regioninfo/1732532427326/Put/seqid=0 2024-11-25T11:00:28,192 INFO [M:0;6bfc37bad115:39111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1df1a5f7727e41ec90d715d8d443e585 2024-11-25T11:00:28,211 DEBUG [M:0;6bfc37bad115:39111 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on db4701ce3c8140caa1eae53dd2bfbfff 2024-11-25T11:00:28,238 DEBUG [M:0;6bfc37bad115:39111 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/db4701ce3c8140caa1eae53dd2bfbfff is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732532425472/Put/seqid=0 2024-11-25T11:00:28,240 INFO [M:0;6bfc37bad115:39111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.27 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/db4701ce3c8140caa1eae53dd2bfbfff 2024-11-25T11:00:28,242 INFO [RS:1;6bfc37bad115:36685 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T11:00:28,242 INFO [RS:1;6bfc37bad115:36685 {}] regionserver.HRegionServer(1031): Exiting; stopping=6bfc37bad115,36685,1732532423630; zookeeper connection closed. 2024-11-25T11:00:28,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:28,242 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36685-0x1007698dd510002, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:28,243 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4875ebd0 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4875ebd0 2024-11-25T11:00:28,243 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-25T11:00:28,266 DEBUG [M:0;6bfc37bad115:39111 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 850112df71be49ed91efdd08dd9af027 2024-11-25T11:00:28,288 DEBUG [M:0;6bfc37bad115:39111 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/850112df71be49ed91efdd08dd9af027 is 69, key is 6bfc37bad115,35003,1732532427210/rs:state/1732532427312/Put/seqid=0 2024-11-25T11:00:28,289 INFO [M:0;6bfc37bad115:39111 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/850112df71be49ed91efdd08dd9af027 2024-11-25T11:00:28,291 INFO [M:0;6bfc37bad115:39111 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 850112df71be49ed91efdd08dd9af027 2024-11-25T11:00:28,292 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1df1a5f7727e41ec90d715d8d443e585 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1df1a5f7727e41ec90d715d8d443e585 2024-11-25T11:00:28,294 INFO [M:0;6bfc37bad115:39111 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1df1a5f7727e41ec90d715d8d443e585, entries=14, sequenceid=102, filesize=5.9 K 2024-11-25T11:00:28,295 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/db4701ce3c8140caa1eae53dd2bfbfff as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/db4701ce3c8140caa1eae53dd2bfbfff 2024-11-25T11:00:28,297 INFO [M:0;6bfc37bad115:39111 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/db4701ce3c8140caa1eae53dd2bfbfff, entries=11, sequenceid=102, filesize=6.5 K 2024-11-25T11:00:28,297 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/850112df71be49ed91efdd08dd9af027 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/850112df71be49ed91efdd08dd9af027 2024-11-25T11:00:28,299 INFO [M:0;6bfc37bad115:39111 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 850112df71be49ed91efdd08dd9af027 2024-11-25T11:00:28,299 INFO [M:0;6bfc37bad115:39111 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e80917b5-7457-dc94-dfcd-3a0b2d3c3cd1/44664e91-aae1-2dc4-4c97-eaf041f4072d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/850112df71be49ed91efdd08dd9af027, entries=3, sequenceid=102, filesize=5.3 K 2024-11-25T11:00:28,300 INFO [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=102, compaction requested=false 2024-11-25T11:00:28,311 INFO [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-25T11:00:28,312 DEBUG [M:0;6bfc37bad115:39111 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732532428159Disabling compacts and flushes for region at 1732532428159Disabling writes for close at 1732532428159Obtaining lock to block concurrent updates at 1732532428159Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732532428159Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40448, getHeapSize=51040, getOffHeapSize=0, getCellsCount=127 at 1732532428160 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732532428160Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732532428160Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732532428190 (+30 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732532428190Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732532428194 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732532428237 (+43 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732532428238 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732532428243 (+5 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732532428287 (+44 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732532428287Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37988d8a: reopening flushed file at 1732532428291 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3053824a: reopening flushed file at 1732532428294 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@223c533b: reopening flushed file at 1732532428297 (+3 ms)Finished flush of dataSize ~39.50 KB/40448, heapSize ~49.61 KB/50800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=102, compaction requested=false at 1732532428300 (+3 ms)Writing region close event to WAL at 1732532428311 (+11 ms)Closed at 1732532428311 2024-11-25T11:00:28,317 INFO [M:0;6bfc37bad115:39111 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-25T11:00:28,317 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-25T11:00:28,317 INFO [M:0;6bfc37bad115:39111 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39111 2024-11-25T11:00:28,321 INFO [M:0;6bfc37bad115:39111 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-25T11:00:28,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:28,423 INFO [M:0;6bfc37bad115:39111 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-25T11:00:28,423 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39111-0x1007698dd510000, quorum=127.0.0.1:61363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-25T11:00:28,426 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-25T11:00:28,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-25T11:00:28,428 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-25T11:00:28,437 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=72 (was 51) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=845 (was 867), ProcessCount=11 (was 11), AvailableMemoryMB=3809 (was 4512)