2024-12-07 06:03:27,630 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 06:03:27,645 main DEBUG Took 0.014089 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-07 06:03:27,646 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-07 06:03:27,646 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-07 06:03:27,647 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-07 06:03:27,648 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,654 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-07 06:03:27,665 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,666 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,666 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,667 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,667 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,668 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,668 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,669 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,669 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,670 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,670 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,670 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,671 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,671 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,672 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,672 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,672 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,673 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,673 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,673 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,674 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-07 06:03:27,674 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,674 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-07 06:03:27,675 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-07 06:03:27,677 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-07 06:03:27,678 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-07 06:03:27,678 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-07 06:03:27,679 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-07 06:03:27,680 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-07 06:03:27,688 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-07 06:03:27,691 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-07 06:03:27,693 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-07 06:03:27,693 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-07 06:03:27,694 main DEBUG createAppenders(={Console}) 2024-12-07 06:03:27,694 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-07 06:03:27,695 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-07 06:03:27,695 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-07 06:03:27,696 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-07 06:03:27,696 main DEBUG OutputStream closed 2024-12-07 06:03:27,696 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-07 06:03:27,697 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-07 06:03:27,697 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-07 06:03:27,771 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-07 06:03:27,773 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-07 06:03:27,774 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-07 06:03:27,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-07 06:03:27,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-07 06:03:27,775 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-07 06:03:27,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-07 06:03:27,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-07 06:03:27,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-07 06:03:27,776 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-07 06:03:27,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-07 06:03:27,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-07 06:03:27,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-07 06:03:27,777 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-07 06:03:27,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-07 06:03:27,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-07 06:03:27,778 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-07 06:03:27,779 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-07 06:03:27,781 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07 06:03:27,781 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-07 06:03:27,782 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-07 06:03:27,782 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-07T06:03:27,794 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-07 06:03:27,796 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-07 06:03:27,797 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-07T06:03:27,988 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c 2024-12-07T06:03:28,007 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=412, ProcessCount=11, AvailableMemoryMB=8801 2024-12-07T06:03:28,092 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-07T06:03:28,173 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:28,187 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/cluster_45baf28a-a6f2-0049-8ce7-74fee15f6d30, deleteOnExit=true 2024-12-07T06:03:28,320 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/cluster_45baf28a-a6f2-0049-8ce7-74fee15f6d30/zookeeper_0, clientPort=64081, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/cluster_45baf28a-a6f2-0049-8ce7-74fee15f6d30/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/cluster_45baf28a-a6f2-0049-8ce7-74fee15f6d30/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T06:03:28,330 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64081 2024-12-07T06:03:28,331 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:28,402 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d with version=8 2024-12-07T06:03:28,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/hbase-staging 2024-12-07T06:03:28,485 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-07T06:03:28,736 INFO [Time-limited test {}] client.ConnectionUtils(128): master/16b2ce7304d2:0 server-side Connection retries=45 2024-12-07T06:03:28,744 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:28,745 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:28,751 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:03:28,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:28,752 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:03:28,873 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T06:03:28,964 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T06:03:29,001 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-07T06:03:29,012 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-07T06:03:29,016 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:03:29,049 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 111903 (auto-detected) 2024-12-07T06:03:29,050 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-07T06:03:29,072 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34461 2024-12-07T06:03:29,093 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34461 connecting to ZooKeeper ensemble=127.0.0.1:64081 2024-12-07T06:03:29,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344610x0, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:03:29,215 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34461-0x101afacd7740000 connected 2024-12-07T06:03:29,323 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:29,324 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:29,333 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:29,336 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d, hbase.cluster.distributed=false 2024-12-07T06:03:29,357 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:03:29,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34461 2024-12-07T06:03:29,362 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34461 2024-12-07T06:03:29,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34461 2024-12-07T06:03:29,363 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34461 2024-12-07T06:03:29,364 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34461 2024-12-07T06:03:29,433 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b2ce7304d2:0 server-side Connection retries=45 2024-12-07T06:03:29,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:29,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:29,435 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:03:29,435 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:29,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:03:29,438 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:03:29,440 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:03:29,441 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39925 2024-12-07T06:03:29,442 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39925 connecting to ZooKeeper ensemble=127.0.0.1:64081 2024-12-07T06:03:29,443 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:29,444 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:29,466 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399250x0, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:03:29,467 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39925-0x101afacd7740001 connected 2024-12-07T06:03:29,467 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:399250x0, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:29,474 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:03:29,488 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.cache 2024-12-07T06:03:29,497 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:03:29,499 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:03:29,501 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:03:29,503 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:03:29,503 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:03:29,510 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:03:29,511 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:03:29,511 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.persistence is complete. 2024-12-07T06:03:29,517 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:03:29,520 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:03:29,527 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:03:29,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39925 2024-12-07T06:03:29,528 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39925 2024-12-07T06:03:29,529 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39925 2024-12-07T06:03:29,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39925 2024-12-07T06:03:29,530 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39925 2024-12-07T06:03:29,548 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b2ce7304d2:0 server-side Connection retries=45 2024-12-07T06:03:29,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:29,548 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:29,549 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:03:29,549 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:29,549 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:03:29,549 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:03:29,550 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:03:29,551 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34429 2024-12-07T06:03:29,553 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:34429 connecting to ZooKeeper ensemble=127.0.0.1:64081 2024-12-07T06:03:29,553 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:29,554 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:29,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:344290x0, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:03:29,584 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:344290x0, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:29,584 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:34429-0x101afacd7740002 connected 2024-12-07T06:03:29,584 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:03:29,586 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.cache 2024-12-07T06:03:29,586 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:03:29,587 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:03:29,587 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:03:29,590 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:03:29,590 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:03:29,590 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:03:29,591 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:03:29,591 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.persistence is complete. 2024-12-07T06:03:29,592 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:03:29,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:03:29,596 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:03:29,598 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34429 2024-12-07T06:03:29,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34429 2024-12-07T06:03:29,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34429 2024-12-07T06:03:29,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34429 2024-12-07T06:03:29,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34429 2024-12-07T06:03:29,621 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;16b2ce7304d2:34461 2024-12-07T06:03:29,622 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:29,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:29,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:29,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:29,639 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:29,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:03:29,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:03:29,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:29,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:29,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:29,667 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T06:03:29,668 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/16b2ce7304d2,34461,1733551408588 from backup master directory 2024-12-07T06:03:29,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:29,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:29,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:29,678 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:29,679 WARN [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:03:29,679 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:29,682 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-07T06:03:29,684 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-07T06:03:29,694 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/hbase.id] with ID: d3268c05-a625-453a-b4bd-11fed9d960e2 2024-12-07T06:03:29,695 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.tmp/hbase.id 2024-12-07T06:03:29,699 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/hbase.id] 2024-12-07T06:03:29,731 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:29,737 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T06:03:29,761 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 21ms. 2024-12-07T06:03:29,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:29,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:29,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:29,813 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:03:29,815 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T06:03:29,851 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-07T06:03:29,856 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:29,906 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store 2024-12-07T06:03:29,929 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-07T06:03:29,932 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:29,933 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:03:29,934 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:29,934 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:29,936 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:03:29,936 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:29,936 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:29,937 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733551409933Disabling compacts and flushes for region at 1733551409933Disabling writes for close at 1733551409936 (+3 ms)Writing region close event to WAL at 1733551409936Closed at 1733551409936 2024-12-07T06:03:29,937 WARN [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/.initializing 2024-12-07T06:03:29,938 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/WALs/16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:29,951 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:03:29,967 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b2ce7304d2%2C34461%2C1733551408588, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/WALs/16b2ce7304d2,34461,1733551408588, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/oldWALs, maxLogs=10 2024-12-07T06:03:30,008 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/WALs/16b2ce7304d2,34461,1733551408588/16b2ce7304d2%2C34461%2C1733551408588.1733551409973 2024-12-07T06:03:30,009 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:30,010 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:30,011 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:30,015 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,016 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,059 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T06:03:30,093 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,095 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:30,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,100 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T06:03:30,101 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:30,101 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,106 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T06:03:30,107 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,107 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:30,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,112 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T06:03:30,113 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:30,114 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,118 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,119 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,123 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,124 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,127 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:03:30,130 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:30,137 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:03:30,138 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61899593, jitterRate=-0.07762418687343597}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:03:30,147 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733551410034Initializing all the Stores at 1733551410035 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551410035Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551410036 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551410037 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551410037Cleaning up temporary data from old regions at 1733551410124 (+87 ms)Region opened successfully at 1733551410146 (+22 ms) 2024-12-07T06:03:30,149 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T06:03:30,187 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c34e25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b2ce7304d2/172.17.0.2:0 2024-12-07T06:03:30,227 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T06:03:30,241 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T06:03:30,242 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T06:03:30,245 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T06:03:30,246 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T06:03:30,252 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-07T06:03:30,252 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T06:03:30,280 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T06:03:30,289 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T06:03:30,336 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T06:03:30,339 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T06:03:30,342 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T06:03:30,352 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T06:03:30,355 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T06:03:30,360 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T06:03:30,369 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T06:03:30,371 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T06:03:30,382 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T06:03:30,397 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T06:03:30,407 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T06:03:30,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:30,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:30,427 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:30,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,428 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,430 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=16b2ce7304d2,34461,1733551408588, sessionid=0x101afacd7740000, setting cluster-up flag (Was=false) 2024-12-07T06:03:30,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,486 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T06:03:30,489 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:30,544 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T06:03:30,547 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:30,555 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T06:03:30,610 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(746): ClusterId : d3268c05-a625-453a-b4bd-11fed9d960e2 2024-12-07T06:03:30,610 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(746): ClusterId : d3268c05-a625-453a-b4bd-11fed9d960e2 2024-12-07T06:03:30,612 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:03:30,612 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:03:30,627 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:30,636 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T06:03:30,638 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:03:30,638 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:03:30,639 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:03:30,639 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:03:30,644 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T06:03:30,650 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:03:30,650 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:03:30,651 DEBUG [RS:0;16b2ce7304d2:39925 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16236027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b2ce7304d2/172.17.0.2:0 2024-12-07T06:03:30,651 DEBUG [RS:1;16b2ce7304d2:34429 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60be0dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b2ce7304d2/172.17.0.2:0 2024-12-07T06:03:30,649 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 16b2ce7304d2,34461,1733551408588 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T06:03:30,658 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:30,658 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:30,658 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:30,659 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:30,659 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/16b2ce7304d2:0, corePoolSize=10, maxPoolSize=10 2024-12-07T06:03:30,659 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,659 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:03:30,660 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,663 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:30,664 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T06:03:30,664 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733551440664 2024-12-07T06:03:30,666 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T06:03:30,666 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;16b2ce7304d2:34429 2024-12-07T06:03:30,667 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T06:03:30,669 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,669 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T06:03:30,670 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:03:30,670 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T06:03:30,670 DEBUG [RS:0;16b2ce7304d2:39925 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;16b2ce7304d2:39925 2024-12-07T06:03:30,670 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:03:30,671 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:03:30,671 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:03:30,671 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:03:30,671 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T06:03:30,671 DEBUG [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:03:30,671 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T06:03:30,672 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T06:03:30,674 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b2ce7304d2,34461,1733551408588 with port=34429, startcode=1733551409547 2024-12-07T06:03:30,674 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b2ce7304d2,34461,1733551408588 with port=39925, startcode=1733551409402 2024-12-07T06:03:30,673 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,676 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T06:03:30,677 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T06:03:30,677 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T06:03:30,681 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T06:03:30,681 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T06:03:30,687 DEBUG [RS:0;16b2ce7304d2:39925 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:03:30,687 DEBUG [RS:1;16b2ce7304d2:34429 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:03:30,690 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.large.0-1733551410682,5,FailOnTimeoutGroup] 2024-12-07T06:03:30,690 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.small.0-1733551410690,5,FailOnTimeoutGroup] 2024-12-07T06:03:30,691 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,691 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T06:03:30,692 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,692 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,697 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T06:03:30,697 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:30,704 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:30,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:03:30,709 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:03:30,709 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:30,709 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:03:30,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:03:30,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:30,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:03:30,722 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:03:30,726 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:30,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:03:30,731 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:03:30,732 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:30,732 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:30,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:03:30,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:30,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:30,734 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:03:30,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:03:30,735 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:03:30,736 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:03:30,744 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:03:30,745 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61366745, jitterRate=-0.08556424081325531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:03:30,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733551410705Initializing all the Stores at 1733551410705Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551410705Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551410705Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551410705Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551410705Cleaning up temporary data from old regions at 1733551410735 (+30 ms)Region opened successfully at 1733551410748 (+13 ms) 2024-12-07T06:03:30,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:03:30,748 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:03:30,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:03:30,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:03:30,749 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:03:30,750 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:03:30,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733551410748Disabling compacts and flushes for region at 1733551410748Disabling writes for close at 1733551410749 (+1 ms)Writing region close event to WAL at 1733551410750 (+1 ms)Closed at 1733551410750 2024-12-07T06:03:30,753 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:30,753 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35065, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:03:30,753 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40443, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:03:30,754 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T06:03:30,761 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34461 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:30,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T06:03:30,764 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34461 {}] master.ServerManager(517): Registering regionserver=16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:30,771 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:03:30,773 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T06:03:30,775 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34461 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:30,775 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34461 {}] master.ServerManager(517): Registering regionserver=16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:30,779 DEBUG [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:30,779 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:30,779 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:03:30,779 DEBUG [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:03:30,780 DEBUG [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:03:30,780 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:03:30,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:30,834 DEBUG [RS:1;16b2ce7304d2:34429 {}] zookeeper.ZKUtil(111): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:30,835 DEBUG [RS:0;16b2ce7304d2:39925 {}] zookeeper.ZKUtil(111): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:30,835 WARN [RS:1;16b2ce7304d2:34429 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:03:30,835 WARN [RS:0;16b2ce7304d2:39925 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:03:30,835 INFO [RS:1;16b2ce7304d2:34429 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:30,835 INFO [RS:0;16b2ce7304d2:39925 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:30,835 DEBUG [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:30,835 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:30,839 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b2ce7304d2,39925,1733551409402] 2024-12-07T06:03:30,839 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b2ce7304d2,34429,1733551409547] 2024-12-07T06:03:30,869 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:03:30,869 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:03:30,884 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:03:30,884 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:03:30,889 INFO [RS:1;16b2ce7304d2:34429 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:03:30,889 INFO [RS:0;16b2ce7304d2:39925 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:03:30,889 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,889 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,891 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:03:30,891 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:03:30,897 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:03:30,897 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:03:30,899 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,899 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,899 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,899 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,899 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,899 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,899 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,899 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b2ce7304d2:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:03:30,900 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b2ce7304d2:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:03:30,900 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,900 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:30,901 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:30,901 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:30,902 DEBUG [RS:1;16b2ce7304d2:34429 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:30,902 DEBUG [RS:0;16b2ce7304d2:39925 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:30,904 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,904 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,905 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34429,1733551409547-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:03:30,905 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,39925,1733551409402-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:03:30,924 WARN [16b2ce7304d2:34461 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T06:03:30,932 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:03:30,932 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:03:30,935 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,39925,1733551409402-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,935 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34429,1733551409547-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,935 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,935 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,935 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.Replication(171): 16b2ce7304d2,39925,1733551409402 started 2024-12-07T06:03:30,935 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.Replication(171): 16b2ce7304d2,34429,1733551409547 started 2024-12-07T06:03:30,961 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,961 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:30,961 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(1482): Serving as 16b2ce7304d2,39925,1733551409402, RpcServer on 16b2ce7304d2/172.17.0.2:39925, sessionid=0x101afacd7740001 2024-12-07T06:03:30,961 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1482): Serving as 16b2ce7304d2,34429,1733551409547, RpcServer on 16b2ce7304d2/172.17.0.2:34429, sessionid=0x101afacd7740002 2024-12-07T06:03:30,962 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:03:30,962 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:03:30,963 DEBUG [RS:0;16b2ce7304d2:39925 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:30,963 DEBUG [RS:1;16b2ce7304d2:34429 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:30,963 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,34429,1733551409547' 2024-12-07T06:03:30,963 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,39925,1733551409402' 2024-12-07T06:03:30,963 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:03:30,963 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:03:30,964 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:03:30,964 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:03:30,965 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:03:30,965 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:03:30,965 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:03:30,965 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:03:30,965 DEBUG [RS:1;16b2ce7304d2:34429 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:30,965 DEBUG [RS:0;16b2ce7304d2:39925 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:30,965 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,34429,1733551409547' 2024-12-07T06:03:30,965 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,39925,1733551409402' 2024-12-07T06:03:30,966 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:03:30,966 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:03:30,966 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:03:30,966 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:03:30,967 DEBUG [RS:0;16b2ce7304d2:39925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:03:30,967 DEBUG [RS:1;16b2ce7304d2:34429 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:03:30,967 INFO [RS:0;16b2ce7304d2:39925 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:03:30,967 INFO [RS:1;16b2ce7304d2:34429 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:03:30,967 INFO [RS:1;16b2ce7304d2:34429 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:03:30,967 INFO [RS:0;16b2ce7304d2:39925 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:03:31,074 INFO [RS:0;16b2ce7304d2:39925 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:03:31,074 INFO [RS:1;16b2ce7304d2:34429 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:03:31,075 INFO [RS:0;16b2ce7304d2:39925 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16b2ce7304d2%2C39925%2C1733551409402, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39925,1733551409402, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs, maxLogs=55 2024-12-07T06:03:31,076 INFO [RS:1;16b2ce7304d2:34429 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16b2ce7304d2%2C34429%2C1733551409547, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,34429,1733551409547, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs, maxLogs=55 2024-12-07T06:03:31,094 INFO [RS:0;16b2ce7304d2:39925 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39925,1733551409402/16b2ce7304d2%2C39925%2C1733551409402.1733551411079 2024-12-07T06:03:31,094 DEBUG [RS:0;16b2ce7304d2:39925 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:31,096 INFO [RS:1;16b2ce7304d2:34429 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,34429,1733551409547/16b2ce7304d2%2C34429%2C1733551409547.1733551411079 2024-12-07T06:03:31,097 DEBUG [RS:1;16b2ce7304d2:34429 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:31,178 DEBUG [16b2ce7304d2:34461 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T06:03:31,190 DEBUG [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(204): Hosts are {16b2ce7304d2=0} racks are {/default-rack=0} 2024-12-07T06:03:31,193 DEBUG [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:03:31,193 DEBUG [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:03:31,194 DEBUG [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:03:31,194 DEBUG [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:03:31,194 INFO [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:03:31,194 INFO [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:03:31,194 DEBUG [16b2ce7304d2:34461 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:03:31,201 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:31,206 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b2ce7304d2,34429,1733551409547, state=OPENING 2024-12-07T06:03:31,249 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T06:03:31,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:31,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:31,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:31,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:31,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:31,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:31,267 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:03:31,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16b2ce7304d2,34429,1733551409547}] 2024-12-07T06:03:31,439 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:03:31,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35817, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:03:31,449 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T06:03:31,449 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:31,450 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T06:03:31,451 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16b2ce7304d2%2C34429%2C1733551409547.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,34429,1733551409547, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs, maxLogs=55 2024-12-07T06:03:31,468 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,34429,1733551409547/16b2ce7304d2%2C34429%2C1733551409547.meta.1733551411452.meta 2024-12-07T06:03:31,468 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:31,469 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:31,470 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T06:03:31,472 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T06:03:31,476 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T06:03:31,479 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T06:03:31,480 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:31,480 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T06:03:31,480 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T06:03:31,482 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:03:31,483 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:03:31,483 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:31,483 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:31,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:03:31,485 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:03:31,485 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:31,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:31,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:03:31,486 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:03:31,486 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:31,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:31,486 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:03:31,487 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:03:31,487 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:31,487 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:31,488 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:03:31,488 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:31,489 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:31,489 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:03:31,490 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:03:31,490 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:03:31,491 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:03:31,491 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63326432, jitterRate=-0.05636262893676758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:03:31,492 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T06:03:31,493 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733551411481Writing region info on filesystem at 1733551411481Initializing all the Stores at 1733551411481Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551411481Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551411482 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551411482Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551411482Cleaning up temporary data from old regions at 1733551411490 (+8 ms)Running coprocessor post-open hooks at 1733551411492 (+2 ms)Region opened successfully at 1733551411493 (+1 ms) 2024-12-07T06:03:31,498 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733551411432 2024-12-07T06:03:31,506 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T06:03:31,507 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T06:03:31,507 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:31,509 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b2ce7304d2,34429,1733551409547, state=OPEN 2024-12-07T06:03:31,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:03:31,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:03:31,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:03:31,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:31,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:31,532 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:31,533 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:31,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T06:03:31,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b2ce7304d2,34429,1733551409547 in 264 msec 2024-12-07T06:03:31,541 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T06:03:31,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 777 msec 2024-12-07T06:03:31,542 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:31,542 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T06:03:31,560 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:03:31,561 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b2ce7304d2,34429,1733551409547, seqNum=-1] 2024-12-07T06:03:31,577 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:03:31,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57071, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:03:31,595 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0070 sec 2024-12-07T06:03:31,595 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733551411595, completionTime=-1 2024-12-07T06:03:31,598 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T06:03:31,598 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T06:03:31,619 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T06:03:31,619 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733551471619 2024-12-07T06:03:31,619 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733551531619 2024-12-07T06:03:31,619 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 21 msec 2024-12-07T06:03:31,622 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T06:03:31,628 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34461,1733551408588-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:31,628 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34461,1733551408588-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:31,628 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34461,1733551408588-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:31,630 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-16b2ce7304d2:34461, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:31,631 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:31,631 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:31,636 DEBUG [master/16b2ce7304d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T06:03:31,657 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.977sec 2024-12-07T06:03:31,659 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T06:03:31,660 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T06:03:31,660 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T06:03:31,661 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T06:03:31,661 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T06:03:31,662 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34461,1733551408588-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:03:31,662 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34461,1733551408588-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T06:03:31,663 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T06:03:31,664 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T06:03:31,664 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,34461,1733551408588-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:31,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:03:31,719 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-07T06:03:31,719 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-07T06:03:31,722 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b2ce7304d2,34461,-1 for getting cluster id 2024-12-07T06:03:31,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T06:03:31,731 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'd3268c05-a625-453a-b4bd-11fed9d960e2' 2024-12-07T06:03:31,734 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T06:03:31,734 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "d3268c05-a625-453a-b4bd-11fed9d960e2" 2024-12-07T06:03:31,736 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e066dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:03:31,736 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b2ce7304d2,34461,-1] 2024-12-07T06:03:31,738 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T06:03:31,740 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:31,741 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53530, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T06:03:31,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:03:31,744 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:03:31,752 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b2ce7304d2,34429,1733551409547, seqNum=-1] 2024-12-07T06:03:31,752 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:03:31,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42370, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:03:31,784 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:31,791 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T06:03:31,796 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:31,798 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@43e5817c 2024-12-07T06:03:31,799 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T06:03:31,801 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T06:03:31,806 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733551411784', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:03:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 2024-12-07T06:03:31,814 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T06:03:31,816 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733551411784" procId is: 4 2024-12-07T06:03:31,816 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:31,818 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T06:03:31,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:31,829 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e4d33082f1c306dde1bf8a56bf37c68a, NAME => 'testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733551411784', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:31,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:31,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(1722): Closing e4d33082f1c306dde1bf8a56bf37c68a, disabling compactions & flushes 2024-12-07T06:03:31,835 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:31,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:31,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. after waiting 0 ms 2024-12-07T06:03:31,835 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:31,835 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:31,836 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733551411784-pool-0 {}] regionserver.HRegion(1676): Region close journal for e4d33082f1c306dde1bf8a56bf37c68a: Waiting for close lock at 1733551411835Disabling compacts and flushes for region at 1733551411835Disabling writes for close at 1733551411835Writing region close event to WAL at 1733551411835Closed at 1733551411835 2024-12-07T06:03:31,837 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T06:03:31,842 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733551411837"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733551411837"}]},"ts":"1733551411837"} 2024-12-07T06:03:31,846 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T06:03:31,848 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T06:03:31,851 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733551411784","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733551411848"}]},"ts":"1733551411848"} 2024-12-07T06:03:31,854 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733551411784, state=ENABLING in hbase:meta 2024-12-07T06:03:31,855 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {16b2ce7304d2=0} racks are {/default-rack=0} 2024-12-07T06:03:31,855 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:03:31,855 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:03:31,855 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:03:31,855 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:03:31,855 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:03:31,855 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:03:31,855 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:03:31,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, ASSIGN}] 2024-12-07T06:03:31,858 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, ASSIGN 2024-12-07T06:03:31,859 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, ASSIGN; state=OFFLINE, location=16b2ce7304d2,39925,1733551409402; forceNewPlan=false, retain=false 2024-12-07T06:03:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:32,012 INFO [16b2ce7304d2:34461 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T06:03:32,013 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e4d33082f1c306dde1bf8a56bf37c68a, regionState=OPENING, regionLocation=16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:32,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, ASSIGN because future has completed 2024-12-07T06:03:32,018 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,39925,1733551409402}] 2024-12-07T06:03:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:32,172 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:03:32,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54571, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:03:32,179 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:32,180 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e4d33082f1c306dde1bf8a56bf37c68a, NAME => 'testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:32,180 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733551411784 e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,180 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:32,181 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,181 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,182 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,185 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d33082f1c306dde1bf8a56bf37c68a columnFamilyName family 2024-12-07T06:03:32,185 DEBUG [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:32,186 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.HStore(327): Store=e4d33082f1c306dde1bf8a56bf37c68a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:32,186 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,187 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,187 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,187 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,187 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,188 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,193 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:03:32,193 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e4d33082f1c306dde1bf8a56bf37c68a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71111829, jitterRate=0.05964882671833038}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:03:32,193 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:32,194 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e4d33082f1c306dde1bf8a56bf37c68a: Running coprocessor pre-open hook at 1733551412181Writing region info on filesystem at 1733551412181Initializing all the Stores at 1733551412181Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551412181Cleaning up temporary data from old regions at 1733551412187 (+6 ms)Running coprocessor post-open hooks at 1733551412193 (+6 ms)Region opened successfully at 1733551412194 (+1 ms) 2024-12-07T06:03:32,195 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a., pid=6, masterSystemTime=1733551412171 2024-12-07T06:03:32,198 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:32,198 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:32,199 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e4d33082f1c306dde1bf8a56bf37c68a, regionState=OPEN, openSeqNum=2, regionLocation=16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:32,203 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,39925,1733551409402 because future has completed 2024-12-07T06:03:32,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T06:03:32,207 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,39925,1733551409402 in 187 msec 2024-12-07T06:03:32,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T06:03:32,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, ASSIGN in 351 msec 2024-12-07T06:03:32,210 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T06:03:32,211 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733551411784","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733551412210"}]},"ts":"1733551412210"} 2024-12-07T06:03:32,214 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733551411784, state=ENABLED in hbase:meta 2024-12-07T06:03:32,215 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T06:03:32,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733551411784 in 407 msec 2024-12-07T06:03:32,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:32,452 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733551411784 completed 2024-12-07T06:03:32,452 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733551411784 get assigned. Timeout = 60000ms 2024-12-07T06:03:32,458 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:03:32,464 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733551411784 assigned to meta. Checking AM states. 2024-12-07T06:03:32,464 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:03:32,465 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733551411784 assigned. 2024-12-07T06:03:32,472 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733551411784', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a., hostname=16b2ce7304d2,39925,1733551409402, seqNum=2] 2024-12-07T06:03:32,474 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:03:32,476 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:03:32,486 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e4d33082f1c306dde1bf8a56bf37c68a 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T06:03:32,506 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:32,542 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/.tmp/family/6392da5af748471885b01daebcd9cb53 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T06:03:32,545 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/.tmp/family/6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:32,578 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/.tmp/family/6392da5af748471885b01daebcd9cb53 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:32,580 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53, delay=999 ms 2024-12-07T06:03:32,582 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T06:03:32,589 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for e4d33082f1c306dde1bf8a56bf37c68a in 102ms, sequenceid=6, compaction requested=false 2024-12-07T06:03:32,590 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733551411784' 2024-12-07T06:03:32,593 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e4d33082f1c306dde1bf8a56bf37c68a: 2024-12-07T06:03:33,583 DEBUG [hfile-prefetch-1733551412580 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6392da5af748471885b01daebcd9cb53_0. 2024-12-07T06:03:33,590 DEBUG [hfile-prefetch-1733551412580 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6392da5af748471885b01daebcd9cb53_117. 2024-12-07T06:03:33,591 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53, with total blocks 2, and data blocks 1 2024-12-07T06:03:33,591 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T06:03:33,592 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T06:03:33,593 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53. Will sleep 100ms and try the verification again. 2024-12-07T06:03:33,603 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=e4d33082f1c306dde1bf8a56bf37c68a, source=16b2ce7304d2,39925,1733551409402, destination=16b2ce7304d2,34429,1733551409547, warming up region on 16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:33,605 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=e4d33082f1c306dde1bf8a56bf37c68a, source=16b2ce7304d2,39925,1733551409402, destination=16b2ce7304d2,34429,1733551409547, running balancer 2024-12-07T06:03:33,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, REOPEN/MOVE 2024-12-07T06:03:33,606 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, REOPEN/MOVE 2024-12-07T06:03:33,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(7855): Warmup {ENCODED => e4d33082f1c306dde1bf8a56bf37c68a, NAME => 'testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:33,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:33,608 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e4d33082f1c306dde1bf8a56bf37c68a, regionState=CLOSING, regionLocation=16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:33,608 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:33,609 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d33082f1c306dde1bf8a56bf37c68a columnFamilyName family 2024-12-07T06:03:33,609 DEBUG [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:33,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, REOPEN/MOVE because future has completed 2024-12-07T06:03:33,612 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-07T06:03:33,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,39925,1733551409402}] 2024-12-07T06:03:33,612 DEBUG [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:33,612 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.HStore(327): Store=e4d33082f1c306dde1bf8a56bf37c68a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(1722): Closing e4d33082f1c306dde1bf8a56bf37c68a, disabling compactions & flushes 2024-12-07T06:03:33,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. after waiting 0 ms 2024-12-07T06:03:33,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,614 DEBUG [StoreFileCloser-e4d33082f1c306dde1bf8a56bf37c68a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:33,614 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34429 {}] regionserver.HRegion(1676): Region close journal for e4d33082f1c306dde1bf8a56bf37c68a: Waiting for close lock at 1733551413613Disabling compacts and flushes for region at 1733551413613Disabling writes for close at 1733551413613Writing region close event to WAL at 1733551413614 (+1 ms)Closed at 1733551413614 2024-12-07T06:03:33,616 WARN [hfile-prefetch-1733551412580 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-07T06:03:33,617 DEBUG [hfile-prefetch-1733551412580 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34461 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-07T06:03:33,693 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T06:03:33,694 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T06:03:33,694 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:33,694 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:03:33,705 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T06:03:33,705 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T06:03:33,706 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:33,706 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:03:33,714 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:03:33,715 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733551413694 2024-12-07T06:03:33,715 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T06:03:33,717 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-07T06:03:33,717 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733551413705 2024-12-07T06:03:33,717 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T06:03:33,773 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:33,773 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-07T06:03:33,774 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing e4d33082f1c306dde1bf8a56bf37c68a, disabling compactions & flushes 2024-12-07T06:03:33,774 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,774 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,775 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. after waiting 0 ms 2024-12-07T06:03:33,775 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,777 DEBUG [StoreFileCloser-e4d33082f1c306dde1bf8a56bf37c68a-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 6392da5af748471885b01daebcd9cb53, starting offset: 0, end offset: 9223372036854775807 2024-12-07T06:03:33,779 DEBUG [StoreFileCloser-e4d33082f1c306dde1bf8a56bf37c68a-family-1 {}] bucket.BucketCache(856): removed key 6392da5af748471885b01daebcd9cb53_0 from back map with offset lock 2094080 in the evict process 2024-12-07T06:03:33,785 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T06:03:33,787 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:33,787 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for e4d33082f1c306dde1bf8a56bf37c68a: Waiting for close lock at 1733551413774Running coprocessor pre-close hooks at 1733551413774Disabling compacts and flushes for region at 1733551413774Disabling writes for close at 1733551413775 (+1 ms)Writing region close event to WAL at 1733551413779 (+4 ms)Running coprocessor post-close hooks at 1733551413786 (+7 ms)Closed at 1733551413787 (+1 ms) 2024-12-07T06:03:33,787 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding e4d33082f1c306dde1bf8a56bf37c68a move to 16b2ce7304d2,34429,1733551409547 record at close sequenceid=6 2024-12-07T06:03:33,791 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:33,791 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e4d33082f1c306dde1bf8a56bf37c68a, regionState=CLOSED 2024-12-07T06:03:33,795 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,39925,1733551409402 because future has completed 2024-12-07T06:03:33,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T06:03:33,799 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,39925,1733551409402 in 184 msec 2024-12-07T06:03:33,800 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, REOPEN/MOVE; state=CLOSED, location=16b2ce7304d2,34429,1733551409547; forceNewPlan=false, retain=false 2024-12-07T06:03:33,815 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T06:03:33,815 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T06:03:33,815 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:33,815 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:03:33,818 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:03:33,818 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733551413815 2024-12-07T06:03:33,819 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T06:03:33,951 INFO [16b2ce7304d2:34461 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T06:03:33,951 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e4d33082f1c306dde1bf8a56bf37c68a, regionState=OPENING, regionLocation=16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:33,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, REOPEN/MOVE because future has completed 2024-12-07T06:03:33,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,34429,1733551409547}] 2024-12-07T06:03:34,114 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,115 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => e4d33082f1c306dde1bf8a56bf37c68a, NAME => 'testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:34,115 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733551411784 e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,116 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:34,116 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,116 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,118 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,119 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e4d33082f1c306dde1bf8a56bf37c68a columnFamilyName family 2024-12-07T06:03:34,119 DEBUG [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:34,120 DEBUG [StoreFileOpener-e4d33082f1c306dde1bf8a56bf37c68a-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53, delay=981 ms 2024-12-07T06:03:34,122 DEBUG [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:34,122 INFO [StoreOpener-e4d33082f1c306dde1bf8a56bf37c68a-1 {}] regionserver.HStore(327): Store=e4d33082f1c306dde1bf8a56bf37c68a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:34,122 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,123 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,123 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,123 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,123 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,127 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,128 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened e4d33082f1c306dde1bf8a56bf37c68a; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59867222, jitterRate=-0.10790887475013733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:03:34,128 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,128 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for e4d33082f1c306dde1bf8a56bf37c68a: Running coprocessor pre-open hook at 1733551414116Writing region info on filesystem at 1733551414117 (+1 ms)Initializing all the Stores at 1733551414117Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551414117Cleaning up temporary data from old regions at 1733551414123 (+6 ms)Running coprocessor post-open hooks at 1733551414128 (+5 ms)Region opened successfully at 1733551414128 2024-12-07T06:03:34,129 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a., pid=9, masterSystemTime=1733551414109 2024-12-07T06:03:34,132 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,132 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=e4d33082f1c306dde1bf8a56bf37c68a, regionState=OPEN, openSeqNum=10, regionLocation=16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:34,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,34429,1733551409547 because future has completed 2024-12-07T06:03:34,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-07T06:03:34,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure e4d33082f1c306dde1bf8a56bf37c68a, server=16b2ce7304d2,34429,1733551409547 in 181 msec 2024-12-07T06:03:34,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733551411784, region=e4d33082f1c306dde1bf8a56bf37c68a, REOPEN/MOVE in 534 msec 2024-12-07T06:03:34,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T06:03:34,159 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:03:34,160 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:34,163 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:34,164 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:34,164 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T06:03:34,164 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T06:03:34,164 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-12-07T06:03:34,164 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=16b2ce7304d2,34461,1733551408588 2024-12-07T06:03:34,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:34,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:34,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:34,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:34,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:34,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:03:34,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:34,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:34,199 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:03:34,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:34,199 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:34,199 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:34,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:34,200 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b2ce7304d2,39925,1733551409402' ***** 2024-12-07T06:03:34,200 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:03:34,200 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b2ce7304d2,34429,1733551409547' ***** 2024-12-07T06:03:34,200 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:03:34,200 INFO [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:03:34,200 INFO [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:03:34,201 INFO [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.persistence 2024-12-07T06:03:34,201 INFO [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/bucket.persistence 2024-12-07T06:03:34,201 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:03:34,201 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:03:34,201 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:03:34,201 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:03:34,201 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:03:34,201 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:03:34,201 INFO [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:03:34,201 INFO [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:03:34,201 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:03:34,201 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:03:34,207 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:03:34,207 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:03:34,210 INFO [regionserver/16b2ce7304d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:34,210 INFO [regionserver/16b2ce7304d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:34,211 DEBUG [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1487): Thread RS:0;16b2ce7304d2:39925 started persisting bucket cache to file 2024-12-07T06:03:34,211 DEBUG [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1487): Thread RS:1;16b2ce7304d2:34429 started persisting bucket cache to file 2024-12-07T06:03:34,211 DEBUG [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:34,211 DEBUG [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:34,211 DEBUG [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:03:34,211 DEBUG [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:03:34,214 DEBUG [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:03:34,214 DEBUG [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733551414211 2024-12-07T06:03:34,214 DEBUG [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:03:34,214 DEBUG [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733551414211 2024-12-07T06:03:34,214 DEBUG [RS:0;16b2ce7304d2:39925 {}] bucket.BucketCache(1509): Thread RS:0;16b2ce7304d2:39925 finished persisting bucket cache to file, renaming 2024-12-07T06:03:34,214 DEBUG [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1509): Thread RS:1;16b2ce7304d2:34429 finished persisting bucket cache to file, renaming 2024-12-07T06:03:34,214 WARN [RS:1;16b2ce7304d2:34429 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-07T06:03:34,215 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:03:34,215 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:03:34,215 INFO [RS:0;16b2ce7304d2:39925 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:03:34,215 INFO [RS:1;16b2ce7304d2:34429 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:03:34,215 INFO [RS:0;16b2ce7304d2:39925 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:03:34,215 INFO [RS:1;16b2ce7304d2:34429 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:03:34,215 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(959): stopping server 16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:34,215 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:03:34,215 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(3091): Received CLOSE for e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,215 INFO [RS:0;16b2ce7304d2:39925 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;16b2ce7304d2:39925. 2024-12-07T06:03:34,215 DEBUG [RS:0;16b2ce7304d2:39925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:34,215 DEBUG [RS:0;16b2ce7304d2:39925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:34,216 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(976): stopping server 16b2ce7304d2,39925,1733551409402; all regions closed. 2024-12-07T06:03:34,216 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(959): stopping server 16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:34,216 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:03:34,216 INFO [RS:1;16b2ce7304d2:34429 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;16b2ce7304d2:34429. 2024-12-07T06:03:34,216 DEBUG [RS:1;16b2ce7304d2:34429 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:34,216 DEBUG [RS:1;16b2ce7304d2:34429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:34,216 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:03:34,216 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:03:34,216 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:03:34,216 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:03:34,216 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e4d33082f1c306dde1bf8a56bf37c68a, disabling compactions & flushes 2024-12-07T06:03:34,216 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T06:03:34,217 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,217 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,217 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. after waiting 0 ms 2024-12-07T06:03:34,217 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,217 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T06:03:34,217 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:03:34,217 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1325): Online Regions={e4d33082f1c306dde1bf8a56bf37c68a=testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a., 1588230740=hbase:meta,,1.1588230740} 2024-12-07T06:03:34,217 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:03:34,217 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:03:34,217 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:03:34,217 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:03:34,217 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:03:34,217 DEBUG [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e4d33082f1c306dde1bf8a56bf37c68a 2024-12-07T06:03:34,217 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-07T06:03:34,218 DEBUG [StoreFileCloser-e4d33082f1c306dde1bf8a56bf37c68a-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/family/6392da5af748471885b01daebcd9cb53 2024-12-07T06:03:34,221 DEBUG [RS:0;16b2ce7304d2:39925 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs 2024-12-07T06:03:34,221 INFO [RS:0;16b2ce7304d2:39925 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b2ce7304d2%2C39925%2C1733551409402:(num 1733551411079) 2024-12-07T06:03:34,221 DEBUG [RS:0;16b2ce7304d2:39925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:34,221 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:34,222 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:03:34,222 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.ChoreService(370): Chore service for: regionserver/16b2ce7304d2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T06:03:34,222 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:03:34,222 INFO [regionserver/16b2ce7304d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:03:34,222 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:03:34,223 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:03:34,223 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:03:34,223 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnRegionMove1733551411784/e4d33082f1c306dde1bf8a56bf37c68a/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T06:03:34,223 INFO [RS:0;16b2ce7304d2:39925 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39925 2024-12-07T06:03:34,223 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,224 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e4d33082f1c306dde1bf8a56bf37c68a: Waiting for close lock at 1733551414216Running coprocessor pre-close hooks at 1733551414216Disabling compacts and flushes for region at 1733551414216Disabling writes for close at 1733551414217 (+1 ms)Writing region close event to WAL at 1733551414219 (+2 ms)Running coprocessor post-close hooks at 1733551414223 (+4 ms)Closed at 1733551414223 2024-12-07T06:03:34,224 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a. 2024-12-07T06:03:34,227 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 0c847d30222f4373ab3fd7ac5a797594 2024-12-07T06:03:34,242 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/info/0c847d30222f4373ab3fd7ac5a797594 is 205, key is testBlockEvictionOnRegionMove1733551411784,,1733551411802.e4d33082f1c306dde1bf8a56bf37c68a./info:regioninfo/1733551414132/Put/seqid=0 2024-12-07T06:03:34,243 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/info/0c847d30222f4373ab3fd7ac5a797594 2024-12-07T06:03:34,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:34,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b2ce7304d2,39925,1733551409402 2024-12-07T06:03:34,243 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:03:34,244 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b2ce7304d2,39925,1733551409402] 2024-12-07T06:03:34,251 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a6dd577f9627438a91b73b41de861744 2024-12-07T06:03:34,260 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b2ce7304d2,39925,1733551409402 already deleted, retry=false 2024-12-07T06:03:34,260 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16b2ce7304d2,39925,1733551409402 expired; onlineServers=1 2024-12-07T06:03:34,263 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/ns/a6dd577f9627438a91b73b41de861744 is 43, key is default/ns:d/1733551411582/Put/seqid=0 2024-12-07T06:03:34,264 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/ns/a6dd577f9627438a91b73b41de861744 2024-12-07T06:03:34,271 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 565695e3605e42a8933cc42e3bf8eb24 2024-12-07T06:03:34,283 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/table/565695e3605e42a8933cc42e3bf8eb24 is 78, key is testBlockEvictionOnRegionMove1733551411784/table:state/1733551412210/Put/seqid=0 2024-12-07T06:03:34,283 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/table/565695e3605e42a8933cc42e3bf8eb24 2024-12-07T06:03:34,286 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/info/0c847d30222f4373ab3fd7ac5a797594 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/0c847d30222f4373ab3fd7ac5a797594 2024-12-07T06:03:34,287 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/0c847d30222f4373ab3fd7ac5a797594, delay=945 ms 2024-12-07T06:03:34,288 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/0c847d30222f4373ab3fd7ac5a797594, entries=15, sequenceid=15, filesize=7.6 K 2024-12-07T06:03:34,289 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/ns/a6dd577f9627438a91b73b41de861744 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/a6dd577f9627438a91b73b41de861744 2024-12-07T06:03:34,290 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/a6dd577f9627438a91b73b41de861744, delay=991 ms 2024-12-07T06:03:34,292 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/a6dd577f9627438a91b73b41de861744, entries=2, sequenceid=15, filesize=5.0 K 2024-12-07T06:03:34,292 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/table/565695e3605e42a8933cc42e3bf8eb24 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/565695e3605e42a8933cc42e3bf8eb24 2024-12-07T06:03:34,294 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/565695e3605e42a8933cc42e3bf8eb24, delay=994 ms 2024-12-07T06:03:34,295 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/565695e3605e42a8933cc42e3bf8eb24, entries=2, sequenceid=15, filesize=5.3 K 2024-12-07T06:03:34,295 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=15, compaction requested=false 2024-12-07T06:03:34,296 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-07T06:03:34,297 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/0c847d30222f4373ab3fd7ac5a797594 2024-12-07T06:03:34,298 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/a6dd577f9627438a91b73b41de861744 2024-12-07T06:03:34,298 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/565695e3605e42a8933cc42e3bf8eb24 2024-12-07T06:03:34,303 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-07T06:03:34,303 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T06:03:34,304 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:03:34,304 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733551414217Running coprocessor pre-close hooks at 1733551414217Disabling compacts and flushes for region at 1733551414217Disabling writes for close at 1733551414217Obtaining lock to block concurrent updates at 1733551414217Preparing flush snapshotting stores in 1588230740 at 1733551414217Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733551414218 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733551414218Flushing 1588230740/info: creating writer at 1733551414218Flushing 1588230740/info: appending metadata at 1733551414241 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733551414241Flushing 1588230740/ns: creating writer at 1733551414246 (+5 ms)Flushing 1588230740/ns: appending metadata at 1733551414262 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733551414262Flushing 1588230740/table: creating writer at 1733551414266 (+4 ms)Flushing 1588230740/table: appending metadata at 1733551414282 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733551414282Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4215221b: reopening flushed file at 1733551414285 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c3b0236: reopening flushed file at 1733551414288 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14bfee1b: reopening flushed file at 1733551414292 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 78ms, sequenceid=15, compaction requested=false at 1733551414296 (+4 ms)Writing region close event to WAL at 1733551414298 (+2 ms)Running coprocessor post-close hooks at 1733551414303 (+5 ms)Closed at 1733551414304 (+1 ms) 2024-12-07T06:03:34,304 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T06:03:34,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:34,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39925-0x101afacd7740001, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:34,353 INFO [RS:0;16b2ce7304d2:39925 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:03:34,353 INFO [RS:0;16b2ce7304d2:39925 {}] regionserver.HRegionServer(1031): Exiting; stopping=16b2ce7304d2,39925,1733551409402; zookeeper connection closed. 2024-12-07T06:03:34,353 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1538cdfd {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1538cdfd 2024-12-07T06:03:34,418 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(976): stopping server 16b2ce7304d2,34429,1733551409547; all regions closed. 2024-12-07T06:03:34,423 DEBUG [RS:1;16b2ce7304d2:34429 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs 2024-12-07T06:03:34,423 INFO [RS:1;16b2ce7304d2:34429 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b2ce7304d2%2C34429%2C1733551409547.meta:.meta(num 1733551411452) 2024-12-07T06:03:34,424 DEBUG [RS:1;16b2ce7304d2:34429 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs 2024-12-07T06:03:34,424 INFO [RS:1;16b2ce7304d2:34429 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b2ce7304d2%2C34429%2C1733551409547:(num 1733551411079) 2024-12-07T06:03:34,425 DEBUG [RS:1;16b2ce7304d2:34429 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:34,425 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:34,425 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:03:34,425 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.ChoreService(370): Chore service for: regionserver/16b2ce7304d2:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T06:03:34,425 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:03:34,425 INFO [regionserver/16b2ce7304d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:03:34,425 INFO [RS:1;16b2ce7304d2:34429 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34429 2024-12-07T06:03:34,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:34,443 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b2ce7304d2,34429,1733551409547 2024-12-07T06:03:34,443 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:03:34,443 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007fbc0c477b20@43b17d56 rejected from java.util.concurrent.ThreadPoolExecutor@6b3eb44[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1360) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-07T06:03:34,451 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b2ce7304d2,34429,1733551409547] 2024-12-07T06:03:34,460 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b2ce7304d2,34429,1733551409547 already deleted, retry=false 2024-12-07T06:03:34,460 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16b2ce7304d2,34429,1733551409547 expired; onlineServers=0 2024-12-07T06:03:34,460 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '16b2ce7304d2,34461,1733551408588' ***** 2024-12-07T06:03:34,460 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T06:03:34,460 INFO [M:0;16b2ce7304d2:34461 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:03:34,460 INFO [M:0;16b2ce7304d2:34461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:03:34,460 DEBUG [M:0;16b2ce7304d2:34461 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T06:03:34,461 DEBUG [M:0;16b2ce7304d2:34461 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T06:03:34,461 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T06:03:34,461 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.small.0-1733551410690 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.small.0-1733551410690,5,FailOnTimeoutGroup] 2024-12-07T06:03:34,461 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.large.0-1733551410682 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.large.0-1733551410682,5,FailOnTimeoutGroup] 2024-12-07T06:03:34,461 INFO [M:0;16b2ce7304d2:34461 {}] hbase.ChoreService(370): Chore service for: master/16b2ce7304d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T06:03:34,461 INFO [M:0;16b2ce7304d2:34461 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:03:34,461 DEBUG [M:0;16b2ce7304d2:34461 {}] master.HMaster(1795): Stopping service threads 2024-12-07T06:03:34,461 INFO [M:0;16b2ce7304d2:34461 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T06:03:34,461 INFO [M:0;16b2ce7304d2:34461 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:03:34,462 INFO [M:0;16b2ce7304d2:34461 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T06:03:34,462 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T06:03:34,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T06:03:34,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:34,468 DEBUG [M:0;16b2ce7304d2:34461 {}] zookeeper.ZKUtil(347): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T06:03:34,469 WARN [M:0;16b2ce7304d2:34461 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T06:03:34,469 INFO [M:0;16b2ce7304d2:34461 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.lastflushedseqids 2024-12-07T06:03:34,484 INFO [M:0;16b2ce7304d2:34461 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T06:03:34,484 INFO [M:0;16b2ce7304d2:34461 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T06:03:34,484 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:03:34,484 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:34,485 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:34,485 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:03:34,485 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:34,485 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-07T06:03:34,494 DEBUG [M:0;16b2ce7304d2:34461 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5d33905ec50047938a47b6467fe6192f 2024-12-07T06:03:34,505 DEBUG [M:0;16b2ce7304d2:34461 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d33905ec50047938a47b6467fe6192f is 82, key is hbase:meta,,1/info:regioninfo/1733551411507/Put/seqid=0 2024-12-07T06:03:34,506 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d33905ec50047938a47b6467fe6192f 2024-12-07T06:03:34,513 DEBUG [M:0;16b2ce7304d2:34461 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7ea88062291c490a859d1ad1eb86477e 2024-12-07T06:03:34,524 DEBUG [M:0;16b2ce7304d2:34461 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ea88062291c490a859d1ad1eb86477e is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733551412217/Put/seqid=0 2024-12-07T06:03:34,525 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ea88062291c490a859d1ad1eb86477e 2024-12-07T06:03:34,533 DEBUG [M:0;16b2ce7304d2:34461 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on faa19d9a26b841658716d2d74c71476d 2024-12-07T06:03:34,545 DEBUG [M:0;16b2ce7304d2:34461 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/faa19d9a26b841658716d2d74c71476d is 69, key is 16b2ce7304d2,34429,1733551409547/rs:state/1733551410767/Put/seqid=0 2024-12-07T06:03:34,546 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/faa19d9a26b841658716d2d74c71476d 2024-12-07T06:03:34,548 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5d33905ec50047938a47b6467fe6192f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d33905ec50047938a47b6467fe6192f 2024-12-07T06:03:34,550 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5d33905ec50047938a47b6467fe6192f, entries=8, sequenceid=84, filesize=5.5 K 2024-12-07T06:03:34,551 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7ea88062291c490a859d1ad1eb86477e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ea88062291c490a859d1ad1eb86477e 2024-12-07T06:03:34,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:34,552 INFO [RS:1;16b2ce7304d2:34429 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:03:34,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:34429-0x101afacd7740002, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:34,552 INFO [RS:1;16b2ce7304d2:34429 {}] regionserver.HRegionServer(1031): Exiting; stopping=16b2ce7304d2,34429,1733551409547; zookeeper connection closed. 2024-12-07T06:03:34,552 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2f4bf5ca {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2f4bf5ca 2024-12-07T06:03:34,552 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-07T06:03:34,553 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7ea88062291c490a859d1ad1eb86477e, entries=9, sequenceid=84, filesize=6.5 K 2024-12-07T06:03:34,553 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/faa19d9a26b841658716d2d74c71476d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/faa19d9a26b841658716d2d74c71476d 2024-12-07T06:03:34,554 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/faa19d9a26b841658716d2d74c71476d, entries=2, sequenceid=84, filesize=5.1 K 2024-12-07T06:03:34,555 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false 2024-12-07T06:03:34,558 INFO [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:34,558 DEBUG [M:0;16b2ce7304d2:34461 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733551414484Disabling compacts and flushes for region at 1733551414484Disabling writes for close at 1733551414485 (+1 ms)Obtaining lock to block concurrent updates at 1733551414485Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733551414485Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733551414486 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733551414486Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733551414486Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733551414505 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733551414505Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733551414508 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733551414524 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733551414524Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733551414527 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733551414545 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733551414545Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da7d62b: reopening flushed file at 1733551414548 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d8bcbb1: reopening flushed file at 1733551414550 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe5f1ef: reopening flushed file at 1733551414553 (+3 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=84, compaction requested=false at 1733551414555 (+2 ms)Writing region close event to WAL at 1733551414558 (+3 ms)Closed at 1733551414558 2024-12-07T06:03:34,559 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:03:34,560 INFO [M:0;16b2ce7304d2:34461 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T06:03:34,560 INFO [M:0;16b2ce7304d2:34461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34461 2024-12-07T06:03:34,560 INFO [M:0;16b2ce7304d2:34461 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:03:34,699 INFO [M:0;16b2ce7304d2:34461 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:03:34,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:34,699 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34461-0x101afacd7740000, quorum=127.0.0.1:64081, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:34,708 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T06:03:34,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T06:03:34,729 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:34,735 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/16b2ce7304d2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/16b2ce7304d2:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733551412580 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733551414287 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/16b2ce7304d2:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733551414120 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/16b2ce7304d2:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733551414290 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 412), ProcessCount=11 (was 11), AvailableMemoryMB=8416 (was 8801) 2024-12-07T06:03:34,741 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=8416 2024-12-07T06:03:34,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/hadoop-log-dir so I do NOT create it in target/test-data/f1480065-fa55-49bc-13e7-f8467330a738 2024-12-07T06:03:34,741 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/hadoop-tmp-dir so I do NOT create it in target/test-data/f1480065-fa55-49bc-13e7-f8467330a738 2024-12-07T06:03:34,741 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:34,744 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/cluster_4d11217a-8940-62f6-ba45-bda5f10289e5, deleteOnExit=true 2024-12-07T06:03:34,748 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/cluster_4d11217a-8940-62f6-ba45-bda5f10289e5/zookeeper_0, clientPort=52064, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/cluster_4d11217a-8940-62f6-ba45-bda5f10289e5/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/cluster_4d11217a-8940-62f6-ba45-bda5f10289e5/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-07T06:03:34,749 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52064 2024-12-07T06:03:34,759 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d with version=8 2024-12-07T06:03:34,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/hbase-staging 2024-12-07T06:03:34,762 INFO [Time-limited test {}] client.ConnectionUtils(128): master/16b2ce7304d2:0 server-side Connection retries=45 2024-12-07T06:03:34,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:34,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:34,762 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:03:34,762 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:34,763 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:03:34,763 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-07T06:03:34,763 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:03:34,763 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41287 2024-12-07T06:03:34,765 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41287 connecting to ZooKeeper ensemble=127.0.0.1:52064 2024-12-07T06:03:34,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:412870x0, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:03:34,812 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41287-0x101afacf09f0000 connected 2024-12-07T06:03:34,950 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:34,950 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:34,953 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:34,954 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d, hbase.cluster.distributed=false 2024-12-07T06:03:34,957 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:03:34,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41287 2024-12-07T06:03:34,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41287 2024-12-07T06:03:34,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41287 2024-12-07T06:03:34,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41287 2024-12-07T06:03:34,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41287 2024-12-07T06:03:34,979 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b2ce7304d2:0 server-side Connection retries=45 2024-12-07T06:03:34,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:34,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:34,979 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:03:34,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:34,979 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:03:34,979 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:03:34,979 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:03:34,980 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33469 2024-12-07T06:03:34,981 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33469 connecting to ZooKeeper ensemble=127.0.0.1:52064 2024-12-07T06:03:34,981 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:34,981 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:34,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:334690x0, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:03:35,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33469-0x101afacf09f0001 connected 2024-12-07T06:03:35,000 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:35,000 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:03:35,001 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.cache 2024-12-07T06:03:35,001 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:03:35,001 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:03:35,001 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:03:35,002 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:03:35,002 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:03:35,002 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:03:35,002 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:03:35,003 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:03:35,003 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence is complete. 2024-12-07T06:03:35,003 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:03:35,005 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:03:35,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33469 2024-12-07T06:03:35,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33469 2024-12-07T06:03:35,006 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33469 2024-12-07T06:03:35,009 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33469 2024-12-07T06:03:35,010 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33469 2024-12-07T06:03:35,026 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b2ce7304d2:0 server-side Connection retries=45 2024-12-07T06:03:35,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:35,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:35,026 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:03:35,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:35,026 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:03:35,026 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:03:35,026 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:03:35,027 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39949 2024-12-07T06:03:35,029 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39949 connecting to ZooKeeper ensemble=127.0.0.1:52064 2024-12-07T06:03:35,029 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:35,030 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:35,049 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:399490x0, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:03:35,050 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:399490x0, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:35,050 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39949-0x101afacf09f0002 connected 2024-12-07T06:03:35,050 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:03:35,051 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.cache 2024-12-07T06:03:35,051 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:03:35,051 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:03:35,051 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:03:35,051 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:03:35,051 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-07T06:03:35,051 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:03:35,052 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:03:35,052 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence is complete. 2024-12-07T06:03:35,052 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:03:35,052 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-07T06:03:35,054 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:03:35,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39949 2024-12-07T06:03:35,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39949 2024-12-07T06:03:35,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39949 2024-12-07T06:03:35,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39949 2024-12-07T06:03:35,059 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39949 2024-12-07T06:03:35,069 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;16b2ce7304d2:41287 2024-12-07T06:03:35,069 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:35,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:35,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:35,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:35,083 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:35,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:03:35,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-07T06:03:35,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,100 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T06:03:35,100 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/16b2ce7304d2,41287,1733551414761 from backup master directory 2024-12-07T06:03:35,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:35,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:35,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:35,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-07T06:03:35,116 WARN [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:03:35,116 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:35,117 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/hbase.id] with ID: c4e2c19d-e177-47f5-bb93-e5a2103540f3 2024-12-07T06:03:35,118 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.tmp/hbase.id 2024-12-07T06:03:35,120 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/hbase.id] 2024-12-07T06:03:35,151 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:35,151 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-07T06:03:35,152 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-12-07T06:03:35,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,171 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:03:35,172 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-07T06:03:35,173 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:35,194 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store 2024-12-07T06:03:35,203 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:35,203 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:03:35,203 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:35,203 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:35,203 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:03:35,203 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:35,204 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:35,204 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733551415203Disabling compacts and flushes for region at 1733551415203Disabling writes for close at 1733551415203Writing region close event to WAL at 1733551415203Closed at 1733551415203 2024-12-07T06:03:35,204 WARN [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/.initializing 2024-12-07T06:03:35,204 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/WALs/16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:35,211 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:03:35,214 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16b2ce7304d2%2C41287%2C1733551414761, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/WALs/16b2ce7304d2,41287,1733551414761, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/oldWALs, maxLogs=10 2024-12-07T06:03:35,230 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/WALs/16b2ce7304d2,41287,1733551414761/16b2ce7304d2%2C41287%2C1733551414761.1733551415214 2024-12-07T06:03:35,231 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:35,231 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:35,231 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:35,231 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,231 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,232 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-07T06:03:35,236 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:35,236 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-07T06:03:35,239 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:35,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-07T06:03:35,243 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:35,243 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-07T06:03:35,246 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,246 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:35,246 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,247 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,247 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,247 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,247 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,247 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:03:35,248 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-07T06:03:35,253 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:03:35,253 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59260186, jitterRate=-0.11695441603660583}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:03:35,254 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733551415231Initializing all the Stores at 1733551415232 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551415232Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551415232Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551415232Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551415232Cleaning up temporary data from old regions at 1733551415247 (+15 ms)Region opened successfully at 1733551415254 (+7 ms) 2024-12-07T06:03:35,254 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-07T06:03:35,258 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1a11fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b2ce7304d2/172.17.0.2:0 2024-12-07T06:03:35,259 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-07T06:03:35,259 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-07T06:03:35,259 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-07T06:03:35,259 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-07T06:03:35,259 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-07T06:03:35,260 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-07T06:03:35,260 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-07T06:03:35,262 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-07T06:03:35,263 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-07T06:03:35,282 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-07T06:03:35,283 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-07T06:03:35,285 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-07T06:03:35,299 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-07T06:03:35,300 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-07T06:03:35,303 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-07T06:03:35,316 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-07T06:03:35,318 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-07T06:03:35,333 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-07T06:03:35,340 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-07T06:03:35,357 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-07T06:03:35,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:35,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:35,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:35,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,375 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=16b2ce7304d2,41287,1733551414761, sessionid=0x101afacf09f0000, setting cluster-up flag (Was=false) 2024-12-07T06:03:35,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,458 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-07T06:03:35,461 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:35,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,491 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:35,541 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-07T06:03:35,542 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:35,543 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-07T06:03:35,544 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:35,545 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-07T06:03:35,545 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-07T06:03:35,545 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 16b2ce7304d2,41287,1733551414761 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-07T06:03:35,547 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:35,548 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:35,548 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:35,548 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=5, maxPoolSize=5 2024-12-07T06:03:35,548 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/16b2ce7304d2:0, corePoolSize=10, maxPoolSize=10 2024-12-07T06:03:35,548 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,548 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:03:35,548 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,549 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733551445549 2024-12-07T06:03:35,550 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-07T06:03:35,550 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-07T06:03:35,550 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-07T06:03:35,550 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-07T06:03:35,550 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-07T06:03:35,550 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-07T06:03:35,550 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:35,550 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-07T06:03:35,550 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,551 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,551 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-07T06:03:35,553 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-07T06:03:35,553 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-07T06:03:35,553 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-07T06:03:35,553 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-07T06:03:35,553 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-07T06:03:35,554 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.large.0-1733551415554,5,FailOnTimeoutGroup] 2024-12-07T06:03:35,554 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.small.0-1733551415554,5,FailOnTimeoutGroup] 2024-12-07T06:03:35,554 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,555 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-07T06:03:35,555 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,555 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,561 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(746): ClusterId : c4e2c19d-e177-47f5-bb93-e5a2103540f3 2024-12-07T06:03:35,561 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(746): ClusterId : c4e2c19d-e177-47f5-bb93-e5a2103540f3 2024-12-07T06:03:35,561 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:03:35,561 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:03:35,569 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-07T06:03:35,570 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:35,575 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:03:35,575 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:03:35,575 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:03:35,575 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:03:35,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:35,577 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:03:35,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:03:35,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:35,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:03:35,583 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:03:35,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:35,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:03:35,586 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:03:35,586 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:35,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:03:35,589 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:03:35,590 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:35,590 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:35,590 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:03:35,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:35,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:35,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:03:35,591 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:03:35,591 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:03:35,591 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:03:35,591 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:03:35,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:03:35,592 DEBUG [RS:1;16b2ce7304d2:39949 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dc91a20, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b2ce7304d2/172.17.0.2:0 2024-12-07T06:03:35,592 DEBUG [RS:0;16b2ce7304d2:33469 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ba5374, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b2ce7304d2/172.17.0.2:0 2024-12-07T06:03:35,599 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:03:35,599 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70929280, jitterRate=0.05692863464355469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:03:35,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733551415576Initializing all the Stores at 1733551415577 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551415577Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551415577Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551415577Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551415577Cleaning up temporary data from old regions at 1733551415591 (+14 ms)Region opened successfully at 1733551415600 (+9 ms) 2024-12-07T06:03:35,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:03:35,600 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:03:35,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:03:35,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:03:35,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:03:35,601 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:03:35,601 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733551415600Disabling compacts and flushes for region at 1733551415600Disabling writes for close at 1733551415600Writing region close event to WAL at 1733551415601 (+1 ms)Closed at 1733551415601 2024-12-07T06:03:35,602 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:35,602 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-07T06:03:35,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-07T06:03:35,603 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:03:35,604 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-07T06:03:35,609 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;16b2ce7304d2:39949 2024-12-07T06:03:35,609 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;16b2ce7304d2:33469 2024-12-07T06:03:35,609 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:03:35,609 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:03:35,609 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:03:35,609 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:03:35,609 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:03:35,609 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:03:35,610 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b2ce7304d2,41287,1733551414761 with port=39949, startcode=1733551415025 2024-12-07T06:03:35,610 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b2ce7304d2,41287,1733551414761 with port=33469, startcode=1733551414978 2024-12-07T06:03:35,611 DEBUG [RS:0;16b2ce7304d2:33469 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:03:35,611 DEBUG [RS:1;16b2ce7304d2:39949 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:03:35,613 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35045, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:03:35,613 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45825, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:03:35,614 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41287 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:35,614 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41287 {}] master.ServerManager(517): Registering regionserver=16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:35,615 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41287 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:35,615 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41287 {}] master.ServerManager(517): Registering regionserver=16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:35,615 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:35,615 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:03:35,615 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:03:35,616 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:35,616 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:03:35,616 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:03:35,632 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:35,670 DEBUG [RS:1;16b2ce7304d2:39949 {}] zookeeper.ZKUtil(111): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:35,670 WARN [RS:1;16b2ce7304d2:39949 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:03:35,670 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b2ce7304d2,39949,1733551415025] 2024-12-07T06:03:35,671 INFO [RS:1;16b2ce7304d2:39949 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:35,671 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b2ce7304d2,33469,1733551414978] 2024-12-07T06:03:35,671 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:35,671 DEBUG [RS:0;16b2ce7304d2:33469 {}] zookeeper.ZKUtil(111): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:35,671 WARN [RS:0;16b2ce7304d2:33469 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:03:35,671 INFO [RS:0;16b2ce7304d2:33469 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:35,672 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:35,679 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:03:35,679 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:03:35,681 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:03:35,681 INFO [RS:0;16b2ce7304d2:33469 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:03:35,681 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,681 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:03:35,682 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:03:35,682 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,682 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,682 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,682 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,682 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b2ce7304d2:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:35,683 DEBUG [RS:0;16b2ce7304d2:33469 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:35,683 INFO [RS:1;16b2ce7304d2:39949 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:03:35,684 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,684 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,684 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:03:35,684 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,684 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,684 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,684 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,684 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,33469,1733551414978-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:03:35,684 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:03:35,684 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,684 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b2ce7304d2:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:35,685 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:35,686 DEBUG [RS:1;16b2ce7304d2:39949 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:35,686 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,686 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,686 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,686 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,686 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,686 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,39949,1733551415025-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:03:35,698 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:03:35,699 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,33469,1733551414978-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,699 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,699 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.Replication(171): 16b2ce7304d2,33469,1733551414978 started 2024-12-07T06:03:35,708 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:03:35,708 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,39949,1733551415025-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,708 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,709 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.Replication(171): 16b2ce7304d2,39949,1733551415025 started 2024-12-07T06:03:35,711 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,711 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1482): Serving as 16b2ce7304d2,33469,1733551414978, RpcServer on 16b2ce7304d2/172.17.0.2:33469, sessionid=0x101afacf09f0001 2024-12-07T06:03:35,711 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:03:35,711 DEBUG [RS:0;16b2ce7304d2:33469 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:35,711 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,33469,1733551414978' 2024-12-07T06:03:35,711 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:03:35,712 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:03:35,712 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:03:35,712 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:03:35,712 DEBUG [RS:0;16b2ce7304d2:33469 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:35,713 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,33469,1733551414978' 2024-12-07T06:03:35,713 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:03:35,713 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:03:35,714 DEBUG [RS:0;16b2ce7304d2:33469 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:03:35,714 INFO [RS:0;16b2ce7304d2:33469 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:03:35,714 INFO [RS:0;16b2ce7304d2:33469 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:03:35,726 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:35,726 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1482): Serving as 16b2ce7304d2,39949,1733551415025, RpcServer on 16b2ce7304d2/172.17.0.2:39949, sessionid=0x101afacf09f0002 2024-12-07T06:03:35,726 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:03:35,726 DEBUG [RS:1;16b2ce7304d2:39949 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:35,726 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,39949,1733551415025' 2024-12-07T06:03:35,726 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:03:35,727 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:03:35,728 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:03:35,728 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:03:35,728 DEBUG [RS:1;16b2ce7304d2:39949 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:35,728 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,39949,1733551415025' 2024-12-07T06:03:35,728 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:03:35,728 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:03:35,729 DEBUG [RS:1;16b2ce7304d2:39949 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:03:35,729 INFO [RS:1;16b2ce7304d2:39949 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:03:35,729 INFO [RS:1;16b2ce7304d2:39949 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:03:35,754 WARN [16b2ce7304d2:41287 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-07T06:03:35,815 INFO [RS:0;16b2ce7304d2:33469 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:03:35,816 INFO [RS:0;16b2ce7304d2:33469 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16b2ce7304d2%2C33469%2C1733551414978, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33469,1733551414978, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs, maxLogs=55 2024-12-07T06:03:35,829 INFO [RS:1;16b2ce7304d2:39949 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:03:35,830 INFO [RS:1;16b2ce7304d2:39949 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16b2ce7304d2%2C39949%2C1733551415025, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39949,1733551415025, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs, maxLogs=55 2024-12-07T06:03:35,838 INFO [RS:0;16b2ce7304d2:33469 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33469,1733551414978/16b2ce7304d2%2C33469%2C1733551414978.1733551415820 2024-12-07T06:03:35,838 DEBUG [RS:0;16b2ce7304d2:33469 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:35,845 INFO [RS:1;16b2ce7304d2:39949 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39949,1733551415025/16b2ce7304d2%2C39949%2C1733551415025.1733551415830 2024-12-07T06:03:35,845 DEBUG [RS:1;16b2ce7304d2:39949 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:36,005 DEBUG [16b2ce7304d2:41287 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-07T06:03:36,005 DEBUG [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(204): Hosts are {16b2ce7304d2=0} racks are {/default-rack=0} 2024-12-07T06:03:36,006 DEBUG [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:03:36,006 DEBUG [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:03:36,006 DEBUG [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:03:36,006 DEBUG [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:03:36,006 INFO [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:03:36,006 INFO [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:03:36,006 DEBUG [16b2ce7304d2:41287 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:03:36,006 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:36,007 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b2ce7304d2,39949,1733551415025, state=OPENING 2024-12-07T06:03:36,041 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-07T06:03:36,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:36,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:36,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:36,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:36,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:36,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:36,056 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-07T06:03:36,056 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16b2ce7304d2,39949,1733551415025}] 2024-12-07T06:03:36,211 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:03:36,218 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54969, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:03:36,222 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-07T06:03:36,222 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:36,223 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-07T06:03:36,224 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16b2ce7304d2%2C39949%2C1733551415025.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39949,1733551415025, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs, maxLogs=55 2024-12-07T06:03:36,237 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,39949,1733551415025/16b2ce7304d2%2C39949%2C1733551415025.meta.1733551416224.meta 2024-12-07T06:03:36,238 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:36,238 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:36,238 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-07T06:03:36,238 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-07T06:03:36,238 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-07T06:03:36,238 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-07T06:03:36,239 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:36,239 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-07T06:03:36,239 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-07T06:03:36,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-07T06:03:36,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-07T06:03:36,241 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:36,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:36,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-07T06:03:36,242 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-07T06:03:36,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:36,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:36,242 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-07T06:03:36,243 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-07T06:03:36,243 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:36,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:36,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-07T06:03:36,244 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-07T06:03:36,244 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:36,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-07T06:03:36,244 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-07T06:03:36,245 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:36,245 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740 2024-12-07T06:03:36,245 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-07T06:03:36,246 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-07T06:03:36,246 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-07T06:03:36,246 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-07T06:03:36,247 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58981397, jitterRate=-0.12110869586467743}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-07T06:03:36,247 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-07T06:03:36,248 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733551416239Writing region info on filesystem at 1733551416239Initializing all the Stores at 1733551416239Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551416239Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551416239Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551416239Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733551416239Cleaning up temporary data from old regions at 1733551416246 (+7 ms)Running coprocessor post-open hooks at 1733551416247 (+1 ms)Region opened successfully at 1733551416248 (+1 ms) 2024-12-07T06:03:36,248 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733551416210 2024-12-07T06:03:36,251 DEBUG [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-07T06:03:36,251 INFO [RS_OPEN_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-07T06:03:36,251 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:36,252 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16b2ce7304d2,39949,1733551415025, state=OPEN 2024-12-07T06:03:36,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:03:36,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:03:36,274 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-07T06:03:36,274 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:36,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:36,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:36,274 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-07T06:03:36,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-07T06:03:36,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16b2ce7304d2,39949,1733551415025 in 218 msec 2024-12-07T06:03:36,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-07T06:03:36,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 675 msec 2024-12-07T06:03:36,282 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-07T06:03:36,282 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-07T06:03:36,287 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:03:36,287 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b2ce7304d2,39949,1733551415025, seqNum=-1] 2024-12-07T06:03:36,287 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:03:36,289 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35565, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:03:36,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 751 msec 2024-12-07T06:03:36,296 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733551416296, completionTime=-1 2024-12-07T06:03:36,296 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-07T06:03:36,296 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-07T06:03:36,299 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-07T06:03:36,299 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733551476299 2024-12-07T06:03:36,299 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733551536299 2024-12-07T06:03:36,299 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-12-07T06:03:36,299 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-07T06:03:36,300 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,41287,1733551414761-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:36,300 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,41287,1733551414761-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:36,300 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,41287,1733551414761-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:36,300 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-16b2ce7304d2:41287, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:36,300 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:36,300 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:36,303 DEBUG [master/16b2ce7304d2:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-07T06:03:36,305 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.189sec 2024-12-07T06:03:36,305 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-07T06:03:36,305 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-07T06:03:36,305 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-07T06:03:36,305 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-07T06:03:36,305 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-07T06:03:36,305 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,41287,1733551414761-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:03:36,306 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,41287,1733551414761-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-07T06:03:36,316 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-07T06:03:36,316 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-07T06:03:36,316 INFO [master/16b2ce7304d2:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,41287,1733551414761-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:36,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:03:36,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16b2ce7304d2,41287,-1 for getting cluster id 2024-12-07T06:03:36,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-07T06:03:36,364 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c4e2c19d-e177-47f5-bb93-e5a2103540f3' 2024-12-07T06:03:36,364 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-07T06:03:36,364 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c4e2c19d-e177-47f5-bb93-e5a2103540f3" 2024-12-07T06:03:36,365 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f832ece, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:03:36,365 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16b2ce7304d2,41287,-1] 2024-12-07T06:03:36,365 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-07T06:03:36,365 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:36,367 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-07T06:03:36,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34126543, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-07T06:03:36,369 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-07T06:03:36,371 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16b2ce7304d2,39949,1733551415025, seqNum=-1] 2024-12-07T06:03:36,371 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:03:36,373 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56472, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:03:36,375 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:36,376 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-07T06:03:36,377 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:36,377 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@260f2f90 2024-12-07T06:03:36,377 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-07T06:03:36,379 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-07T06:03:36,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41287 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733551416375', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-07T06:03:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41287 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 2024-12-07T06:03:36,382 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-07T06:03:36,383 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:36,383 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41287 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733551416375" procId is: 4 2024-12-07T06:03:36,383 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-07T06:03:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41287 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:36,393 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c919d5900559d11b43b0a1e9fbe3ad9e, NAME => 'testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733551416375', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:36,399 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:36,399 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(1722): Closing c919d5900559d11b43b0a1e9fbe3ad9e, disabling compactions & flushes 2024-12-07T06:03:36,399 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:36,399 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:36,399 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. after waiting 0 ms 2024-12-07T06:03:36,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:36,400 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:36,400 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733551416375-pool-0 {}] regionserver.HRegion(1676): Region close journal for c919d5900559d11b43b0a1e9fbe3ad9e: Waiting for close lock at 1733551416399Disabling compacts and flushes for region at 1733551416399Disabling writes for close at 1733551416400 (+1 ms)Writing region close event to WAL at 1733551416400Closed at 1733551416400 2024-12-07T06:03:36,401 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 execute state=CREATE_TABLE_ADD_TO_META 2024-12-07T06:03:36,401 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733551416401"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733551416401"}]},"ts":"1733551416401"} 2024-12-07T06:03:36,404 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-07T06:03:36,405 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-07T06:03:36,405 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733551416375","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733551416405"}]},"ts":"1733551416405"} 2024-12-07T06:03:36,408 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733551416375, state=ENABLING in hbase:meta 2024-12-07T06:03:36,408 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {16b2ce7304d2=0} racks are {/default-rack=0} 2024-12-07T06:03:36,409 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-07T06:03:36,409 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-07T06:03:36,409 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-07T06:03:36,409 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-07T06:03:36,409 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-07T06:03:36,409 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-07T06:03:36,409 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-07T06:03:36,409 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN}] 2024-12-07T06:03:36,411 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN 2024-12-07T06:03:36,411 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN; state=OFFLINE, location=16b2ce7304d2,33469,1733551414978; forceNewPlan=false, retain=false 2024-12-07T06:03:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41287 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:36,562 INFO [16b2ce7304d2:41287 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-07T06:03:36,563 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c919d5900559d11b43b0a1e9fbe3ad9e, regionState=OPENING, regionLocation=16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:36,570 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN because future has completed 2024-12-07T06:03:36,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c919d5900559d11b43b0a1e9fbe3ad9e, server=16b2ce7304d2,33469,1733551414978}] 2024-12-07T06:03:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41287 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:36,725 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-07T06:03:36,727 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41353, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-07T06:03:36,730 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:36,730 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => c919d5900559d11b43b0a1e9fbe3ad9e, NAME => 'testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:36,731 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733551416375 c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,731 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:36,731 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,731 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,732 INFO [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,737 INFO [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c919d5900559d11b43b0a1e9fbe3ad9e columnFamilyName family 2024-12-07T06:03:36,738 DEBUG [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:36,738 INFO [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] regionserver.HStore(327): Store=c919d5900559d11b43b0a1e9fbe3ad9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:36,738 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,740 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,740 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,740 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,740 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,741 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,747 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-07T06:03:36,748 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened c919d5900559d11b43b0a1e9fbe3ad9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72437560, jitterRate=0.07940375804901123}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:03:36,748 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:36,749 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for c919d5900559d11b43b0a1e9fbe3ad9e: Running coprocessor pre-open hook at 1733551416731Writing region info on filesystem at 1733551416731Initializing all the Stores at 1733551416732 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551416732Cleaning up temporary data from old regions at 1733551416740 (+8 ms)Running coprocessor post-open hooks at 1733551416748 (+8 ms)Region opened successfully at 1733551416749 (+1 ms) 2024-12-07T06:03:36,749 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e., pid=6, masterSystemTime=1733551416724 2024-12-07T06:03:36,752 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:36,752 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:36,752 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=c919d5900559d11b43b0a1e9fbe3ad9e, regionState=OPEN, openSeqNum=2, regionLocation=16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:36,754 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure c919d5900559d11b43b0a1e9fbe3ad9e, server=16b2ce7304d2,33469,1733551414978 because future has completed 2024-12-07T06:03:36,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-07T06:03:36,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure c919d5900559d11b43b0a1e9fbe3ad9e, server=16b2ce7304d2,33469,1733551414978 in 185 msec 2024-12-07T06:03:36,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-07T06:03:36,760 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN in 349 msec 2024-12-07T06:03:36,760 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-07T06:03:36,761 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733551416375","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733551416760"}]},"ts":"1733551416760"} 2024-12-07T06:03:36,763 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733551416375, state=ENABLED in hbase:meta 2024-12-07T06:03:36,764 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 execute state=CREATE_TABLE_POST_OPERATION 2024-12-07T06:03:36,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733551416375 in 384 msec 2024-12-07T06:03:36,982 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-07T06:03:37,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41287 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-07T06:03:37,009 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733551416375 completed 2024-12-07T06:03:37,009 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733551416375 get assigned. Timeout = 60000ms 2024-12-07T06:03:37,009 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:03:37,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733551416375 assigned to meta. Checking AM states. 2024-12-07T06:03:37,012 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-07T06:03:37,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733551416375 assigned. 2024-12-07T06:03:37,015 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733551416375', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e., hostname=16b2ce7304d2,33469,1733551414978, seqNum=2] 2024-12-07T06:03:37,016 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-07T06:03:37,018 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-07T06:03:37,022 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing c919d5900559d11b43b0a1e9fbe3ad9e 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-07T06:03:37,030 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 94c8abd03e41491194fee3d3ebd15943 2024-12-07T06:03:37,041 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/.tmp/family/94c8abd03e41491194fee3d3ebd15943 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-07T06:03:37,041 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/.tmp/family/94c8abd03e41491194fee3d3ebd15943 2024-12-07T06:03:37,043 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/.tmp/family/94c8abd03e41491194fee3d3ebd15943 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943 2024-12-07T06:03:37,044 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943, delay=998 ms 2024-12-07T06:03:37,045 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943, entries=2, sequenceid=6, filesize=4.9 K 2024-12-07T06:03:37,046 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for c919d5900559d11b43b0a1e9fbe3ad9e in 24ms, sequenceid=6, compaction requested=false 2024-12-07T06:03:37,046 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733551416375' 2024-12-07T06:03:37,047 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for c919d5900559d11b43b0a1e9fbe3ad9e: 2024-12-07T06:03:38,044 DEBUG [hfile-prefetch-1733551412580 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 94c8abd03e41491194fee3d3ebd15943_0. 2024-12-07T06:03:38,045 DEBUG [hfile-prefetch-1733551412580 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 94c8abd03e41491194fee3d3ebd15943_117. 2024-12-07T06:03:38,046 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943, with total blocks 2, and data blocks 1 2024-12-07T06:03:38,046 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-07T06:03:38,046 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-07T06:03:38,047 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943. Will sleep 100ms and try the verification again. 2024-12-07T06:03:38,047 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;16b2ce7304d2:33469,5,FailOnTimeoutGroup] 2024-12-07T06:03:38,048 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b2ce7304d2,33469,1733551414978' ***** 2024-12-07T06:03:38,048 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-12-07T06:03:38,048 INFO [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:03:38,087 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:03:38,087 INFO [regionserver/16b2ce7304d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:38,145 INFO [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence 2024-12-07T06:03:38,145 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:03:38,145 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:03:38,145 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:03:38,146 INFO [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:03:38,146 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:03:38,147 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943, with total blocks 2, and data blocks 1 2024-12-07T06:03:38,147 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-12-07T06:03:38,147 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2293): found block for file 94c8abd03e41491194fee3d3ebd15943 in the backing map. Acquiring read lock for offset 0 2024-12-07T06:03:38,148 DEBUG [hfile-prefetch-1733551412580 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943 has now been fully cached. 2024-12-07T06:03:38,148 DEBUG [hfile-prefetch-1733551412580 {}] hfile.PrefetchExecutor(121): Prefetch completed for 94c8abd03e41491194fee3d3ebd15943 2024-12-07T06:03:38,154 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-07T06:03:38,154 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-07T06:03:38,155 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:38,155 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:03:38,156 DEBUG [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1487): Thread RS:0;16b2ce7304d2:33469 started persisting bucket cache to file 2024-12-07T06:03:38,156 DEBUG [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:38,156 DEBUG [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-07T06:03:38,160 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:03:38,160 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733551418154 2024-12-07T06:03:38,160 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-07T06:03:38,162 DEBUG [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-07T06:03:38,162 DEBUG [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733551418156 2024-12-07T06:03:38,162 DEBUG [RS:0;16b2ce7304d2:33469 {}] bucket.BucketCache(1509): Thread RS:0;16b2ce7304d2:33469 finished persisting bucket cache to file, renaming 2024-12-07T06:03:38,163 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:03:38,163 INFO [RS:0;16b2ce7304d2:33469 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:03:38,163 INFO [RS:0;16b2ce7304d2:33469 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:03:38,163 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(3091): Received CLOSE for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,164 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(959): stopping server 16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:38,164 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:03:38,164 INFO [RS:0;16b2ce7304d2:33469 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;16b2ce7304d2:33469. 2024-12-07T06:03:38,164 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:03:38,164 DEBUG [RS:0;16b2ce7304d2:33469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:38,164 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c919d5900559d11b43b0a1e9fbe3ad9e, disabling compactions & flushes 2024-12-07T06:03:38,164 DEBUG [RS:0;16b2ce7304d2:33469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:38,164 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,164 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,164 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-07T06:03:38,164 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1325): Online Regions={c919d5900559d11b43b0a1e9fbe3ad9e=testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.} 2024-12-07T06:03:38,164 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. after waiting 0 ms 2024-12-07T06:03:38,165 DEBUG [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1351): Waiting on c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,165 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,173 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-07T06:03:38,173 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,174 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c919d5900559d11b43b0a1e9fbe3ad9e: Waiting for close lock at 1733551418164Running coprocessor pre-close hooks at 1733551418164Disabling compacts and flushes for region at 1733551418164Disabling writes for close at 1733551418165 (+1 ms)Writing region close event to WAL at 1733551418166 (+1 ms)Running coprocessor post-close hooks at 1733551418173 (+7 ms)Closed at 1733551418173 2024-12-07T06:03:38,174 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,365 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(976): stopping server 16b2ce7304d2,33469,1733551414978; all regions closed. 2024-12-07T06:03:38,367 DEBUG [RS:0;16b2ce7304d2:33469 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs 2024-12-07T06:03:38,367 INFO [RS:0;16b2ce7304d2:33469 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b2ce7304d2%2C33469%2C1733551414978:(num 1733551415820) 2024-12-07T06:03:38,367 DEBUG [RS:0;16b2ce7304d2:33469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:38,367 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:38,367 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:03:38,367 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.ChoreService(370): Chore service for: regionserver/16b2ce7304d2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-07T06:03:38,368 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:03:38,368 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:03:38,368 INFO [regionserver/16b2ce7304d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:03:38,368 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:03:38,368 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:03:38,368 INFO [RS:0;16b2ce7304d2:33469 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33469 2024-12-07T06:03:38,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b2ce7304d2,33469,1733551414978 2024-12-07T06:03:38,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:38,385 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:03:38,393 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b2ce7304d2,33469,1733551414978] 2024-12-07T06:03:38,401 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b2ce7304d2,33469,1733551414978 already deleted, retry=false 2024-12-07T06:03:38,401 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 16b2ce7304d2,33469,1733551414978 on 16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:38,407 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 16b2ce7304d2,33469,1733551414978, splitWal=true, meta=false 2024-12-07T06:03:38,410 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 16b2ce7304d2,33469,1733551414978 (carryingMeta=false) 16b2ce7304d2,33469,1733551414978/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6b5464ef[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-07T06:03:38,410 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 16b2ce7304d2,33469,1733551414978, splitWal=true, meta=false 2024-12-07T06:03:38,412 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 16b2ce7304d2,33469,1733551414978 had 1 regions 2024-12-07T06:03:38,414 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 16b2ce7304d2,33469,1733551414978, splitWal=true, meta=false, isMeta: false 2024-12-07T06:03:38,414 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33469,1733551414978-splitting 2024-12-07T06:03:38,415 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33469,1733551414978-splitting dir is empty, no logs to split. 2024-12-07T06:03:38,415 INFO [PEWorker-2 {}] master.SplitWALManager(105): 16b2ce7304d2,33469,1733551414978 WAL count=0, meta=false 2024-12-07T06:03:38,418 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33469,1733551414978-splitting dir is empty, no logs to split. 2024-12-07T06:03:38,418 INFO [PEWorker-2 {}] master.SplitWALManager(105): 16b2ce7304d2,33469,1733551414978 WAL count=0, meta=false 2024-12-07T06:03:38,418 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 16b2ce7304d2,33469,1733551414978 WAL splitting is done? wals=0, meta=false 2024-12-07T06:03:38,419 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 16b2ce7304d2,33469,1733551414978 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33469,1733551414978-splitting does not exist 2024-12-07T06:03:38,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN}] 2024-12-07T06:03:38,423 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN 2024-12-07T06:03:38,423 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-07T06:03:38,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:38,493 INFO [RS:0;16b2ce7304d2:33469 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:03:38,493 INFO [RS:0;16b2ce7304d2:33469 {}] regionserver.HRegionServer(1031): Exiting; stopping=16b2ce7304d2,33469,1733551414978; zookeeper connection closed. 2024-12-07T06:03:38,493 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33469-0x101afacf09f0001, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:38,494 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@274e0810 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@274e0810 2024-12-07T06:03:38,562 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16b2ce7304d2:0 server-side Connection retries=45 2024-12-07T06:03:38,563 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:38,563 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:38,563 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-07T06:03:38,563 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-07T06:03:38,563 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-07T06:03:38,563 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-07T06:03:38,563 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-07T06:03:38,564 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33281 2024-12-07T06:03:38,565 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33281 connecting to ZooKeeper ensemble=127.0.0.1:52064 2024-12-07T06:03:38,565 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:38,565 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:38,574 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c919d5900559d11b43b0a1e9fbe3ad9e, regionState=OPENING, regionLocation=16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:38,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN because future has completed 2024-12-07T06:03:38,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c919d5900559d11b43b0a1e9fbe3ad9e, server=16b2ce7304d2,39949,1733551415025}] 2024-12-07T06:03:38,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:332810x0, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-07T06:03:38,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33281-0x101afacf09f0003 connected 2024-12-07T06:03:38,591 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-07T06:03:38,591 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-07T06:03:38,593 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-07T06:03:38,594 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.cache 2024-12-07T06:03:38,594 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-07T06:03:38,594 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-07T06:03:38,594 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-07T06:03:38,594 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-07T06:03:38,595 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-07T06:03:38,595 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-07T06:03:38,596 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-07T06:03:38,596 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-07T06:03:38,598 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 1 of chunks with blockCount = 1. 2024-12-07T06:03:38,599 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-07T06:03:38,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33281 2024-12-07T06:03:38,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33281 2024-12-07T06:03:38,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33281 2024-12-07T06:03:38,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33281 2024-12-07T06:03:38,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33281 2024-12-07T06:03:38,601 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-07T06:03:38,601 DEBUG [Thread-201 {}] bucket.BucketCache(1584): Cached File Entry:<94c8abd03e41491194fee3d3ebd15943,> 2024-12-07T06:03:38,602 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-07T06:03:38,602 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 1 2024-12-07T06:03:38,602 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence is complete. 2024-12-07T06:03:38,602 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(746): ClusterId : c4e2c19d-e177-47f5-bb93-e5a2103540f3 2024-12-07T06:03:38,602 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-07T06:03:38,610 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-07T06:03:38,611 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-07T06:03:38,619 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-07T06:03:38,619 DEBUG [RS:2;16b2ce7304d2:33281 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb6266b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16b2ce7304d2/172.17.0.2:0 2024-12-07T06:03:38,634 DEBUG [RS:2;16b2ce7304d2:33281 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;16b2ce7304d2:33281 2024-12-07T06:03:38,634 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-07T06:03:38,634 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-07T06:03:38,634 DEBUG [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-07T06:03:38,635 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(2659): reportForDuty to master=16b2ce7304d2,41287,1733551414761 with port=33281, startcode=1733551418562 2024-12-07T06:03:38,635 DEBUG [RS:2;16b2ce7304d2:33281 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-07T06:03:38,637 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48289, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-07T06:03:38,637 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41287 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:38,638 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41287 {}] master.ServerManager(517): Registering regionserver=16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:38,639 DEBUG [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d 2024-12-07T06:03:38,639 DEBUG [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-07T06:03:38,639 DEBUG [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-07T06:03:38,649 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:38,649 DEBUG [RS:2;16b2ce7304d2:33281 {}] zookeeper.ZKUtil(111): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:38,649 WARN [RS:2;16b2ce7304d2:33281 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-07T06:03:38,650 INFO [RS:2;16b2ce7304d2:33281 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-07T06:03:38,650 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16b2ce7304d2,33281,1733551418562] 2024-12-07T06:03:38,650 DEBUG [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:38,654 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-07T06:03:38,656 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-07T06:03:38,656 INFO [RS:2;16b2ce7304d2:33281 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-07T06:03:38,656 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,656 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-07T06:03:38,657 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-07T06:03:38,657 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16b2ce7304d2:0, corePoolSize=2, maxPoolSize=2 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,657 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,658 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,658 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,658 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16b2ce7304d2:0, corePoolSize=1, maxPoolSize=1 2024-12-07T06:03:38,658 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:38,658 DEBUG [RS:2;16b2ce7304d2:33281 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16b2ce7304d2:0, corePoolSize=3, maxPoolSize=3 2024-12-07T06:03:38,658 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,658 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,658 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,658 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,658 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,658 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,33281,1733551418562-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-07T06:03:38,672 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-07T06:03:38,672 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=16b2ce7304d2,33281,1733551418562-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,673 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,673 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.Replication(171): 16b2ce7304d2,33281,1733551418562 started 2024-12-07T06:03:38,685 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-07T06:03:38,685 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(1482): Serving as 16b2ce7304d2,33281,1733551418562, RpcServer on 16b2ce7304d2/172.17.0.2:33281, sessionid=0x101afacf09f0003 2024-12-07T06:03:38,685 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-07T06:03:38,685 DEBUG [RS:2;16b2ce7304d2:33281 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:38,685 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,33281,1733551418562' 2024-12-07T06:03:38,686 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-07T06:03:38,686 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-07T06:03:38,687 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-07T06:03:38,687 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-07T06:03:38,687 DEBUG [RS:2;16b2ce7304d2:33281 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:38,687 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16b2ce7304d2,33281,1733551418562' 2024-12-07T06:03:38,687 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-07T06:03:38,687 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-07T06:03:38,688 DEBUG [RS:2;16b2ce7304d2:33281 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-07T06:03:38,688 INFO [RS:2;16b2ce7304d2:33281 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-07T06:03:38,688 INFO [RS:2;16b2ce7304d2:33281 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-07T06:03:38,732 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,732 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => c919d5900559d11b43b0a1e9fbe3ad9e, NAME => 'testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.', STARTKEY => '', ENDKEY => ''} 2024-12-07T06:03:38,733 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733551416375 c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,733 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-07T06:03:38,733 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,733 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,734 INFO [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,736 INFO [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c919d5900559d11b43b0a1e9fbe3ad9e columnFamilyName family 2024-12-07T06:03:38,736 DEBUG [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-07T06:03:38,739 DEBUG [StoreFileOpener-c919d5900559d11b43b0a1e9fbe3ad9e-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943, delay=994 ms 2024-12-07T06:03:38,741 DEBUG [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943 2024-12-07T06:03:38,741 INFO [StoreOpener-c919d5900559d11b43b0a1e9fbe3ad9e-1 {}] regionserver.HStore(327): Store=c919d5900559d11b43b0a1e9fbe3ad9e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-07T06:03:38,741 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,743 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,743 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,743 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,744 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,745 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,746 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened c919d5900559d11b43b0a1e9fbe3ad9e; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69041924, jitterRate=0.02880483865737915}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-07T06:03:38,746 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:38,747 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for c919d5900559d11b43b0a1e9fbe3ad9e: Running coprocessor pre-open hook at 1733551418733Writing region info on filesystem at 1733551418733Initializing all the Stores at 1733551418734 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733551418734Cleaning up temporary data from old regions at 1733551418744 (+10 ms)Running coprocessor post-open hooks at 1733551418746 (+2 ms)Region opened successfully at 1733551418747 (+1 ms) 2024-12-07T06:03:38,748 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e., pid=9, masterSystemTime=1733551418729 2024-12-07T06:03:38,750 DEBUG [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,750 INFO [RS_OPEN_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:38,751 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=c919d5900559d11b43b0a1e9fbe3ad9e, regionState=OPEN, openSeqNum=10, regionLocation=16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:38,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure c919d5900559d11b43b0a1e9fbe3ad9e, server=16b2ce7304d2,39949,1733551415025 because future has completed 2024-12-07T06:03:38,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-07T06:03:38,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure c919d5900559d11b43b0a1e9fbe3ad9e, server=16b2ce7304d2,39949,1733551415025 in 176 msec 2024-12-07T06:03:38,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-07T06:03:38,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733551416375, region=c919d5900559d11b43b0a1e9fbe3ad9e, ASSIGN in 335 msec 2024-12-07T06:03:38,757 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 16b2ce7304d2,33469,1733551414978 after splitting done 2024-12-07T06:03:38,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 16b2ce7304d2,33469,1733551414978, splitWal=true, meta=false in 355 msec 2024-12-07T06:03:38,788 INFO [RS:2;16b2ce7304d2:33281 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-07T06:03:38,789 INFO [RS:2;16b2ce7304d2:33281 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16b2ce7304d2%2C33281%2C1733551418562, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33281,1733551418562, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs, maxLogs=55 2024-12-07T06:03:38,807 INFO [RS:2;16b2ce7304d2:33281 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/WALs/16b2ce7304d2,33281,1733551418562/16b2ce7304d2%2C33281%2C1733551418562.1733551418790 2024-12-07T06:03:38,808 DEBUG [RS:2;16b2ce7304d2:33281 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-07T06:03:38,998 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-07T06:03:38,998 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-07T06:03:39,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T06:03:39,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-07T06:03:39,000 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-07T06:03:39,000 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-07T06:03:39,001 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733551416375 2024-12-07T06:03:39,001 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1733551416375 Metrics about Tables on a single HBase RegionServer 2024-12-07T06:03:39,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-07T06:03:39,186 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:03:39,186 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:39,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:39,187 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:39,187 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-07T06:03:39,188 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-07T06:03:39,188 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=788292443, stopped=false 2024-12-07T06:03:39,188 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=16b2ce7304d2,41287,1733551414761 2024-12-07T06:03:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:39,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-07T06:03:39,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:39,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:39,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:39,235 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:03:39,235 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-07T06:03:39,235 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:39,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:39,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:39,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:39,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-07T06:03:39,236 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b2ce7304d2,39949,1733551415025' ***** 2024-12-07T06:03:39,236 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:03:39,236 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16b2ce7304d2,33281,1733551418562' ***** 2024-12-07T06:03:39,236 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-07T06:03:39,236 INFO [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:03:39,236 INFO [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1738): Disabling cache 2024-12-07T06:03:39,236 INFO [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence 2024-12-07T06:03:39,236 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:03:39,236 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:03:39,236 INFO [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/f1480065-fa55-49bc-13e7-f8467330a738/bucket.persistence 2024-12-07T06:03:39,236 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-07T06:03:39,236 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:03:39,236 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-07T06:03:39,237 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-07T06:03:39,237 INFO [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:03:39,237 INFO [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-07T06:03:39,237 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:03:39,237 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-07T06:03:39,247 DEBUG [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1487): Thread RS:2;16b2ce7304d2:33281 started persisting bucket cache to file 2024-12-07T06:03:39,247 DEBUG [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1487): Thread RS:1;16b2ce7304d2:39949 started persisting bucket cache to file 2024-12-07T06:03:39,247 DEBUG [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:39,247 DEBUG [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-07T06:03:39,247 DEBUG [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 1, chunkSize: 10000 2024-12-07T06:03:39,247 DEBUG [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-07T06:03:39,250 DEBUG [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-07T06:03:39,250 DEBUG [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733551419247 2024-12-07T06:03:39,250 DEBUG [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-12-07T06:03:39,250 DEBUG [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1733551419247 2024-12-07T06:03:39,250 DEBUG [RS:1;16b2ce7304d2:39949 {}] bucket.BucketCache(1509): Thread RS:1;16b2ce7304d2:39949 finished persisting bucket cache to file, renaming 2024-12-07T06:03:39,250 DEBUG [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1509): Thread RS:2;16b2ce7304d2:33281 finished persisting bucket cache to file, renaming 2024-12-07T06:03:39,250 WARN [RS:2;16b2ce7304d2:33281 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-07T06:03:39,250 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:03:39,250 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-07T06:03:39,250 INFO [RS:1;16b2ce7304d2:39949 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:03:39,250 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:03:39,250 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-07T06:03:39,250 INFO [RS:1;16b2ce7304d2:39949 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:03:39,250 INFO [RS:2;16b2ce7304d2:33281 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-07T06:03:39,250 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(3091): Received CLOSE for c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:39,250 INFO [RS:2;16b2ce7304d2:33281 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-07T06:03:39,251 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(959): stopping server 16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:39,251 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:03:39,251 INFO [RS:2;16b2ce7304d2:33281 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;16b2ce7304d2:33281. 2024-12-07T06:03:39,251 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(959): stopping server 16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:39,251 DEBUG [RS:2;16b2ce7304d2:33281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:39,251 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:03:39,251 DEBUG [RS:2;16b2ce7304d2:33281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:39,251 INFO [RS:1;16b2ce7304d2:39949 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;16b2ce7304d2:39949. 2024-12-07T06:03:39,251 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(976): stopping server 16b2ce7304d2,33281,1733551418562; all regions closed. 2024-12-07T06:03:39,251 DEBUG [RS:1;16b2ce7304d2:39949 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-07T06:03:39,251 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:03:39,251 DEBUG [RS:1;16b2ce7304d2:39949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:39,251 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c919d5900559d11b43b0a1e9fbe3ad9e, disabling compactions & flushes 2024-12-07T06:03:39,251 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:03:39,251 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:39,251 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:03:39,251 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:39,251 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:03:39,251 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. after waiting 0 ms 2024-12-07T06:03:39,251 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-07T06:03:39,251 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:39,252 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-07T06:03:39,252 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1325): Online Regions={c919d5900559d11b43b0a1e9fbe3ad9e=testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e., 1588230740=hbase:meta,,1.1588230740} 2024-12-07T06:03:39,252 DEBUG [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, c919d5900559d11b43b0a1e9fbe3ad9e 2024-12-07T06:03:39,252 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-07T06:03:39,252 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-07T06:03:39,252 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-07T06:03:39,252 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-07T06:03:39,252 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-07T06:03:39,252 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-07T06:03:39,252 DEBUG [RS:2;16b2ce7304d2:33281 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs 2024-12-07T06:03:39,252 INFO [RS:2;16b2ce7304d2:33281 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b2ce7304d2%2C33281%2C1733551418562:(num 1733551418790) 2024-12-07T06:03:39,252 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-12-07T06:03:39,252 DEBUG [RS:2;16b2ce7304d2:33281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:39,252 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:39,252 DEBUG [StoreFileCloser-c919d5900559d11b43b0a1e9fbe3ad9e-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/family/94c8abd03e41491194fee3d3ebd15943 2024-12-07T06:03:39,253 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:03:39,253 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.ChoreService(370): Chore service for: regionserver/16b2ce7304d2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T06:03:39,253 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-07T06:03:39,253 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-07T06:03:39,253 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-07T06:03:39,253 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:03:39,253 INFO [RS:2;16b2ce7304d2:33281 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33281 2024-12-07T06:03:39,253 INFO [regionserver/16b2ce7304d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:03:39,259 INFO [regionserver/16b2ce7304d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:39,261 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/default/testBlockEvictionOnGracefulStop1733551416375/c919d5900559d11b43b0a1e9fbe3ad9e/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-07T06:03:39,261 INFO [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:39,261 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c919d5900559d11b43b0a1e9fbe3ad9e: Waiting for close lock at 1733551419251Running coprocessor pre-close hooks at 1733551419251Disabling compacts and flushes for region at 1733551419251Disabling writes for close at 1733551419251Writing region close event to WAL at 1733551419253 (+2 ms)Running coprocessor post-close hooks at 1733551419261 (+8 ms)Closed at 1733551419261 2024-12-07T06:03:39,262 DEBUG [RS_CLOSE_REGION-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e. 2024-12-07T06:03:39,265 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 58102d8adf6b4343a06e3d4438cdcc4e 2024-12-07T06:03:39,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b2ce7304d2,33281,1733551418562 2024-12-07T06:03:39,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:39,268 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:03:39,277 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/info/58102d8adf6b4343a06e3d4438cdcc4e is 209, key is testBlockEvictionOnGracefulStop1733551416375,,1733551416380.c919d5900559d11b43b0a1e9fbe3ad9e./info:regioninfo/1733551418751/Put/seqid=0 2024-12-07T06:03:39,278 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/info/58102d8adf6b4343a06e3d4438cdcc4e 2024-12-07T06:03:39,282 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b2ce7304d2,33281,1733551418562] 2024-12-07T06:03:39,286 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on aa3f120ebf4c49a1b6125f1e343a41c1 2024-12-07T06:03:39,292 INFO [regionserver/16b2ce7304d2:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:39,292 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b2ce7304d2,33281,1733551418562 already deleted, retry=false 2024-12-07T06:03:39,293 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16b2ce7304d2,33281,1733551418562 expired; onlineServers=1 2024-12-07T06:03:39,297 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/ns/aa3f120ebf4c49a1b6125f1e343a41c1 is 43, key is default/ns:d/1733551416290/Put/seqid=0 2024-12-07T06:03:39,298 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/ns/aa3f120ebf4c49a1b6125f1e343a41c1 2024-12-07T06:03:39,305 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on dcbef4a289dc4956aa72756ccc6598dd 2024-12-07T06:03:39,316 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/table/dcbef4a289dc4956aa72756ccc6598dd is 80, key is testBlockEvictionOnGracefulStop1733551416375/table:state/1733551416760/Put/seqid=0 2024-12-07T06:03:39,316 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/table/dcbef4a289dc4956aa72756ccc6598dd 2024-12-07T06:03:39,318 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/info/58102d8adf6b4343a06e3d4438cdcc4e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/58102d8adf6b4343a06e3d4438cdcc4e 2024-12-07T06:03:39,320 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/58102d8adf6b4343a06e3d4438cdcc4e, delay=903 ms 2024-12-07T06:03:39,320 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/58102d8adf6b4343a06e3d4438cdcc4e, entries=14, sequenceid=13, filesize=7.5 K 2024-12-07T06:03:39,321 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/ns/aa3f120ebf4c49a1b6125f1e343a41c1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/aa3f120ebf4c49a1b6125f1e343a41c1 2024-12-07T06:03:39,322 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/aa3f120ebf4c49a1b6125f1e343a41c1, delay=950 ms 2024-12-07T06:03:39,323 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/aa3f120ebf4c49a1b6125f1e343a41c1, entries=2, sequenceid=13, filesize=5.0 K 2024-12-07T06:03:39,323 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/.tmp/table/dcbef4a289dc4956aa72756ccc6598dd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/dcbef4a289dc4956aa72756ccc6598dd 2024-12-07T06:03:39,324 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/dcbef4a289dc4956aa72756ccc6598dd, delay=995 ms 2024-12-07T06:03:39,325 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/dcbef4a289dc4956aa72756ccc6598dd, entries=2, sequenceid=13, filesize=5.3 K 2024-12-07T06:03:39,326 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false 2024-12-07T06:03:39,327 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/info/58102d8adf6b4343a06e3d4438cdcc4e 2024-12-07T06:03:39,327 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/ns/aa3f120ebf4c49a1b6125f1e343a41c1 2024-12-07T06:03:39,329 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/table/dcbef4a289dc4956aa72756ccc6598dd 2024-12-07T06:03:39,334 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-12-07T06:03:39,334 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-07T06:03:39,334 INFO [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-07T06:03:39,334 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733551419252Running coprocessor pre-close hooks at 1733551419252Disabling compacts and flushes for region at 1733551419252Disabling writes for close at 1733551419252Obtaining lock to block concurrent updates at 1733551419252Preparing flush snapshotting stores in 1588230740 at 1733551419252Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1733551419253 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733551419253Flushing 1588230740/info: creating writer at 1733551419253Flushing 1588230740/info: appending metadata at 1733551419277 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733551419277Flushing 1588230740/ns: creating writer at 1733551419280 (+3 ms)Flushing 1588230740/ns: appending metadata at 1733551419297 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1733551419297Flushing 1588230740/table: creating writer at 1733551419300 (+3 ms)Flushing 1588230740/table: appending metadata at 1733551419315 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733551419315Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@665145cd: reopening flushed file at 1733551419318 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@52219ec: reopening flushed file at 1733551419321 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c860c55: reopening flushed file at 1733551419323 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false at 1733551419326 (+3 ms)Writing region close event to WAL at 1733551419329 (+3 ms)Running coprocessor post-close hooks at 1733551419334 (+5 ms)Closed at 1733551419334 2024-12-07T06:03:39,335 DEBUG [RS_CLOSE_META-regionserver/16b2ce7304d2:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-07T06:03:39,382 INFO [RS:2;16b2ce7304d2:33281 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:03:39,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:39,382 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33281-0x101afacf09f0003, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:39,382 INFO [RS:2;16b2ce7304d2:33281 {}] regionserver.HRegionServer(1031): Exiting; stopping=16b2ce7304d2,33281,1733551418562; zookeeper connection closed. 2024-12-07T06:03:39,383 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@7d03258c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@7d03258c 2024-12-07T06:03:39,452 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(976): stopping server 16b2ce7304d2,39949,1733551415025; all regions closed. 2024-12-07T06:03:39,457 DEBUG [RS:1;16b2ce7304d2:39949 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs 2024-12-07T06:03:39,458 INFO [RS:1;16b2ce7304d2:39949 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b2ce7304d2%2C39949%2C1733551415025.meta:.meta(num 1733551416224) 2024-12-07T06:03:39,460 DEBUG [RS:1;16b2ce7304d2:39949 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/oldWALs 2024-12-07T06:03:39,460 INFO [RS:1;16b2ce7304d2:39949 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16b2ce7304d2%2C39949%2C1733551415025:(num 1733551415830) 2024-12-07T06:03:39,460 DEBUG [RS:1;16b2ce7304d2:39949 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-07T06:03:39,460 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.LeaseManager(133): Closed leases 2024-12-07T06:03:39,460 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:03:39,460 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.ChoreService(370): Chore service for: regionserver/16b2ce7304d2:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-07T06:03:39,460 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:03:39,460 INFO [regionserver/16b2ce7304d2:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:03:39,461 INFO [RS:1;16b2ce7304d2:39949 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39949 2024-12-07T06:03:39,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-07T06:03:39,484 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16b2ce7304d2,39949,1733551415025 2024-12-07T06:03:39,485 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:03:39,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16b2ce7304d2,39949,1733551415025] 2024-12-07T06:03:39,501 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16b2ce7304d2,39949,1733551415025 already deleted, retry=false 2024-12-07T06:03:39,501 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16b2ce7304d2,39949,1733551415025 expired; onlineServers=0 2024-12-07T06:03:39,501 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '16b2ce7304d2,41287,1733551414761' ***** 2024-12-07T06:03:39,502 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-07T06:03:39,502 INFO [M:0;16b2ce7304d2:41287 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-07T06:03:39,502 INFO [M:0;16b2ce7304d2:41287 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-07T06:03:39,502 DEBUG [M:0;16b2ce7304d2:41287 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-07T06:03:39,503 DEBUG [M:0;16b2ce7304d2:41287 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-07T06:03:39,503 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-07T06:03:39,503 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.large.0-1733551415554 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.large.0-1733551415554,5,FailOnTimeoutGroup] 2024-12-07T06:03:39,503 DEBUG [master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.small.0-1733551415554 {}] cleaner.HFileCleaner(306): Exit Thread[master/16b2ce7304d2:0:becomeActiveMaster-HFileCleaner.small.0-1733551415554,5,FailOnTimeoutGroup] 2024-12-07T06:03:39,503 INFO [M:0;16b2ce7304d2:41287 {}] hbase.ChoreService(370): Chore service for: master/16b2ce7304d2:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-07T06:03:39,504 INFO [M:0;16b2ce7304d2:41287 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-07T06:03:39,504 DEBUG [M:0;16b2ce7304d2:41287 {}] master.HMaster(1795): Stopping service threads 2024-12-07T06:03:39,504 INFO [M:0;16b2ce7304d2:41287 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-07T06:03:39,504 INFO [M:0;16b2ce7304d2:41287 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-07T06:03:39,505 INFO [M:0;16b2ce7304d2:41287 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-07T06:03:39,505 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-07T06:03:39,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-07T06:03:39,515 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-07T06:03:39,515 DEBUG [M:0;16b2ce7304d2:41287 {}] zookeeper.ZKUtil(347): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-07T06:03:39,515 WARN [M:0;16b2ce7304d2:41287 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-07T06:03:39,516 INFO [M:0;16b2ce7304d2:41287 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/.lastflushedseqids 2024-12-07T06:03:39,521 INFO [M:0;16b2ce7304d2:41287 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-07T06:03:39,522 INFO [M:0;16b2ce7304d2:41287 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-07T06:03:39,522 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-07T06:03:39,522 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:39,522 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:39,522 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-07T06:03:39,522 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:39,522 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-12-07T06:03:39,530 DEBUG [M:0;16b2ce7304d2:41287 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 374cca4effe54027abe7453a8c659e6b 2024-12-07T06:03:39,540 DEBUG [M:0;16b2ce7304d2:41287 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/374cca4effe54027abe7453a8c659e6b is 82, key is hbase:meta,,1/info:regioninfo/1733551416251/Put/seqid=0 2024-12-07T06:03:39,541 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/374cca4effe54027abe7453a8c659e6b 2024-12-07T06:03:39,547 DEBUG [M:0;16b2ce7304d2:41287 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3a0d99be15ae429eabe0d30c85e42714 2024-12-07T06:03:39,559 DEBUG [M:0;16b2ce7304d2:41287 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a0d99be15ae429eabe0d30c85e42714 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733551416765/Put/seqid=0 2024-12-07T06:03:39,559 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a0d99be15ae429eabe0d30c85e42714 2024-12-07T06:03:39,566 DEBUG [M:0;16b2ce7304d2:41287 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7bccfa9bbe8449a9bd83eaed59104789 2024-12-07T06:03:39,578 DEBUG [M:0;16b2ce7304d2:41287 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7bccfa9bbe8449a9bd83eaed59104789 is 69, key is 16b2ce7304d2,33281,1733551418562/rs:state/1733551418638/Put/seqid=0 2024-12-07T06:03:39,578 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7bccfa9bbe8449a9bd83eaed59104789 2024-12-07T06:03:39,580 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7bccfa9bbe8449a9bd83eaed59104789 2024-12-07T06:03:39,580 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/374cca4effe54027abe7453a8c659e6b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/374cca4effe54027abe7453a8c659e6b 2024-12-07T06:03:39,582 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/374cca4effe54027abe7453a8c659e6b, entries=8, sequenceid=83, filesize=5.5 K 2024-12-07T06:03:39,582 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3a0d99be15ae429eabe0d30c85e42714 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a0d99be15ae429eabe0d30c85e42714 2024-12-07T06:03:39,583 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3a0d99be15ae429eabe0d30c85e42714, entries=9, sequenceid=83, filesize=6.4 K 2024-12-07T06:03:39,583 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7bccfa9bbe8449a9bd83eaed59104789 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7bccfa9bbe8449a9bd83eaed59104789 2024-12-07T06:03:39,585 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7bccfa9bbe8449a9bd83eaed59104789 2024-12-07T06:03:39,585 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e3b293ee-1dfd-4b77-1fab-18e1489fcd0c/86e1cf81-9c72-5282-75bb-79d3af835c7d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7bccfa9bbe8449a9bd83eaed59104789, entries=3, sequenceid=83, filesize=5.3 K 2024-12-07T06:03:39,585 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 63ms, sequenceid=83, compaction requested=false 2024-12-07T06:03:39,587 INFO [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-07T06:03:39,587 DEBUG [M:0;16b2ce7304d2:41287 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733551419522Disabling compacts and flushes for region at 1733551419522Disabling writes for close at 1733551419522Obtaining lock to block concurrent updates at 1733551419522Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733551419522Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1733551419523 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733551419523Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733551419523Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733551419540 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733551419540Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733551419542 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733551419558 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733551419558Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733551419561 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733551419577 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733551419577Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@393db907: reopening flushed file at 1733551419580 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4030fecb: reopening flushed file at 1733551419582 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f838863: reopening flushed file at 1733551419583 (+1 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 63ms, sequenceid=83, compaction requested=false at 1733551419585 (+2 ms)Writing region close event to WAL at 1733551419587 (+2 ms)Closed at 1733551419587 2024-12-07T06:03:39,588 INFO [M:0;16b2ce7304d2:41287 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-07T06:03:39,588 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-07T06:03:39,588 INFO [M:0;16b2ce7304d2:41287 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41287 2024-12-07T06:03:39,588 INFO [M:0;16b2ce7304d2:41287 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-07T06:03:39,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:39,593 INFO [RS:1;16b2ce7304d2:39949 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:03:39,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39949-0x101afacf09f0002, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:39,593 INFO [RS:1;16b2ce7304d2:39949 {}] regionserver.HRegionServer(1031): Exiting; stopping=16b2ce7304d2,39949,1733551415025; zookeeper connection closed. 2024-12-07T06:03:39,593 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36aa16f1 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36aa16f1 2024-12-07T06:03:39,594 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-07T06:03:39,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:39,716 INFO [M:0;16b2ce7304d2:41287 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-07T06:03:39,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41287-0x101afacf09f0000, quorum=127.0.0.1:52064, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-07T06:03:39,721 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-07T06:03:39,723 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-07T06:03:39,723 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-07T06:03:39,731 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=387 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=8318 (was 8416)