2024-11-16 08:58:59,949 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 08:58:59,968 main DEBUG Took 0.016494 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-16 08:58:59,968 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-16 08:58:59,969 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-16 08:58:59,969 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-16 08:58:59,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:58:59,978 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-16 08:58:59,990 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:58:59,992 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:58:59,992 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:58:59,993 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:58:59,994 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:58:59,994 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:58:59,996 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:58:59,996 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:58:59,997 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:58:59,997 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:58:59,999 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:58:59,999 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:59:00,000 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,000 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:59:00,001 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,001 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:59:00,002 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,002 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:59:00,003 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,003 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:59:00,004 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,004 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:59:00,005 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,006 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-16 08:59:00,006 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,007 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-16 08:59:00,008 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-16 08:59:00,010 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-16 08:59:00,013 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-16 08:59:00,013 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-16 08:59:00,015 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-16 08:59:00,016 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-16 08:59:00,026 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-16 08:59:00,029 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-16 08:59:00,031 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-16 08:59:00,032 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-16 08:59:00,032 main DEBUG createAppenders(={Console}) 2024-11-16 08:59:00,033 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-16 08:59:00,034 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-16 08:59:00,034 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-16 08:59:00,035 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-16 08:59:00,035 main DEBUG OutputStream closed 2024-11-16 08:59:00,036 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-16 08:59:00,036 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-16 08:59:00,036 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-16 08:59:00,124 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-16 08:59:00,126 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-16 08:59:00,127 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-16 08:59:00,128 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-16 08:59:00,129 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-16 08:59:00,129 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-16 08:59:00,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-16 08:59:00,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-16 08:59:00,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-16 08:59:00,130 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-16 08:59:00,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-16 08:59:00,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-16 08:59:00,131 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-16 08:59:00,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-16 08:59:00,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-16 08:59:00,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-16 08:59:00,132 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-16 08:59:00,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-16 08:59:00,135 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16 08:59:00,136 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-16 08:59:00,136 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-16 08:59:00,136 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-16T08:59:00,149 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-16 08:59:00,151 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-16 08:59:00,151 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-16T08:59:00,380 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f 2024-11-16T08:59:00,403 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=100, ProcessCount=11, AvailableMemoryMB=6961 2024-11-16T08:59:00,487 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-16T08:59:00,567 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:00,580 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/cluster_67c24e43-e6f3-04dd-4855-0fd5d4598476, deleteOnExit=true 2024-11-16T08:59:00,705 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/cluster_67c24e43-e6f3-04dd-4855-0fd5d4598476/zookeeper_0, clientPort=51196, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/cluster_67c24e43-e6f3-04dd-4855-0fd5d4598476/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/cluster_67c24e43-e6f3-04dd-4855-0fd5d4598476/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T08:59:00,716 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51196 2024-11-16T08:59:00,717 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:00,788 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 with version=8 2024-11-16T08:59:00,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/hbase-staging 2024-11-16T08:59:00,883 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-16T08:59:01,121 INFO [Time-limited test {}] client.ConnectionUtils(128): master/16bf49104b8f:0 server-side Connection retries=45 2024-11-16T08:59:01,130 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,131 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,138 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T08:59:01,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,139 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T08:59:01,268 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T08:59:01,348 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T08:59:01,388 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-16T08:59:01,399 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-16T08:59:01,403 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T08:59:01,440 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 92371 (auto-detected) 2024-11-16T08:59:01,442 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-16T08:59:01,467 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44307 2024-11-16T08:59:01,488 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44307 connecting to ZooKeeper ensemble=127.0.0.1:51196 2024-11-16T08:59:01,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:443070x0, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T08:59:01,564 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44307-0x101442801920000 connected 2024-11-16T08:59:01,644 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:01,646 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:01,656 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:01,660 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8, hbase.cluster.distributed=false 2024-11-16T08:59:01,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T08:59:01,691 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44307 2024-11-16T08:59:01,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44307 2024-11-16T08:59:01,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44307 2024-11-16T08:59:01,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44307 2024-11-16T08:59:01,693 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44307 2024-11-16T08:59:01,782 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16bf49104b8f:0 server-side Connection retries=45 2024-11-16T08:59:01,783 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,784 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T08:59:01,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,784 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T08:59:01,787 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T08:59:01,790 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T08:59:01,791 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40505 2024-11-16T08:59:01,792 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40505 connecting to ZooKeeper ensemble=127.0.0.1:51196 2024-11-16T08:59:01,793 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:01,794 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:01,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405050x0, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T08:59:01,813 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:405050x0, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:01,813 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40505-0x101442801920001 connected 2024-11-16T08:59:01,821 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T08:59:01,832 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.cache 2024-11-16T08:59:01,837 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T08:59:01,839 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T08:59:01,840 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T08:59:01,842 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T08:59:01,842 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T08:59:01,847 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T08:59:01,852 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T08:59:01,853 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T08:59:01,853 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.persistence is complete. 2024-11-16T08:59:01,855 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T08:59:01,860 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T08:59:01,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40505 2024-11-16T08:59:01,862 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40505 2024-11-16T08:59:01,863 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40505 2024-11-16T08:59:01,866 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40505 2024-11-16T08:59:01,867 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40505 2024-11-16T08:59:01,885 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16bf49104b8f:0 server-side Connection retries=45 2024-11-16T08:59:01,885 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,886 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T08:59:01,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:01,886 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T08:59:01,886 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T08:59:01,887 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T08:59:01,888 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37041 2024-11-16T08:59:01,891 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37041 connecting to ZooKeeper ensemble=127.0.0.1:51196 2024-11-16T08:59:01,891 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:01,892 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:01,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370410x0, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T08:59:01,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370410x0, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:01,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37041-0x101442801920002 connected 2024-11-16T08:59:01,921 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T08:59:01,923 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.cache 2024-11-16T08:59:01,923 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T08:59:01,924 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T08:59:01,924 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T08:59:01,924 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T08:59:01,925 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T08:59:01,925 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T08:59:01,925 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T08:59:01,925 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T08:59:01,926 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.persistence is complete. 2024-11-16T08:59:01,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T08:59:01,930 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T08:59:01,933 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37041 2024-11-16T08:59:01,934 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37041 2024-11-16T08:59:01,936 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37041 2024-11-16T08:59:01,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37041 2024-11-16T08:59:01,939 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37041 2024-11-16T08:59:01,955 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;16bf49104b8f:44307 2024-11-16T08:59:01,956 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/16bf49104b8f,44307,1731747540960 2024-11-16T08:59:01,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:01,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:01,971 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:01,974 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/16bf49104b8f,44307,1731747540960 2024-11-16T08:59:02,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T08:59:02,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T08:59:02,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,005 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T08:59:02,007 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/16bf49104b8f,44307,1731747540960 from backup master directory 2024-11-16T08:59:02,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:02,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/16bf49104b8f,44307,1731747540960 2024-11-16T08:59:02,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:02,019 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:02,019 WARN [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T08:59:02,020 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=16bf49104b8f,44307,1731747540960 2024-11-16T08:59:02,022 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-16T08:59:02,024 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-16T08:59:02,032 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/hbase.id] with ID: 00677e86-245f-40a3-8b82-c52b135b3607 2024-11-16T08:59:02,032 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.tmp/hbase.id 2024-11-16T08:59:02,037 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/hbase.id] 2024-11-16T08:59:02,071 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:02,078 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T08:59:02,104 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-16T08:59:02,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,118 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,145 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T08:59:02,148 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T08:59:02,187 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-16T08:59:02,192 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:02,241 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store 2024-11-16T08:59:02,261 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-16T08:59:02,265 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:02,292 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T08:59:02,292 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:02,293 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:02,295 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T08:59:02,295 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:02,295 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:02,296 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731747542292Disabling compacts and flushes for region at 1731747542292Disabling writes for close at 1731747542295 (+3 ms)Writing region close event to WAL at 1731747542295Closed at 1731747542295 2024-11-16T08:59:02,296 WARN [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/.initializing 2024-11-16T08:59:02,296 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/WALs/16bf49104b8f,44307,1731747540960 2024-11-16T08:59:02,310 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T08:59:02,326 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16bf49104b8f%2C44307%2C1731747540960, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/WALs/16bf49104b8f,44307,1731747540960, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/oldWALs, maxLogs=10 2024-11-16T08:59:02,369 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/WALs/16bf49104b8f,44307,1731747540960/16bf49104b8f%2C44307%2C1731747540960.1731747542331 2024-11-16T08:59:02,370 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:02,370 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:02,371 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:02,375 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,376 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,413 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T08:59:02,446 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:02,448 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:02,449 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T08:59:02,454 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:02,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:02,455 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T08:59:02,461 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:02,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:02,461 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T08:59:02,466 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:02,466 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:02,467 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,471 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,472 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,476 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,477 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,480 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T08:59:02,483 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:02,490 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T08:59:02,491 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63559040, jitterRate=-0.05289649963378906}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T08:59:02,498 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731747542390Initializing all the Stores at 1731747542390Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747542391 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747542392 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747542393 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747542393Cleaning up temporary data from old regions at 1731747542477 (+84 ms)Region opened successfully at 1731747542498 (+21 ms) 2024-11-16T08:59:02,500 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T08:59:02,538 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e924572, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16bf49104b8f/172.17.0.2:0 2024-11-16T08:59:02,579 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T08:59:02,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T08:59:02,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T08:59:02,599 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T08:59:02,600 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T08:59:02,607 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-16T08:59:02,608 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T08:59:02,640 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T08:59:02,651 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T08:59:02,720 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T08:59:02,723 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T08:59:02,725 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T08:59:02,735 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T08:59:02,737 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T08:59:02,742 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T08:59:02,753 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T08:59:02,756 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T08:59:02,768 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T08:59:02,790 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T08:59:02,801 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T08:59:02,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:02,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:02,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:02,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,823 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=16bf49104b8f,44307,1731747540960, sessionid=0x101442801920000, setting cluster-up flag (Was=false) 2024-11-16T08:59:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,879 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T08:59:02,882 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16bf49104b8f,44307,1731747540960 2024-11-16T08:59:02,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:02,928 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T08:59:02,931 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16bf49104b8f,44307,1731747540960 2024-11-16T08:59:02,936 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T08:59:02,944 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(746): ClusterId : 00677e86-245f-40a3-8b82-c52b135b3607 2024-11-16T08:59:02,944 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(746): ClusterId : 00677e86-245f-40a3-8b82-c52b135b3607 2024-11-16T08:59:02,946 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T08:59:02,946 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T08:59:02,956 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T08:59:02,956 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T08:59:02,956 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T08:59:02,956 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T08:59:02,963 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T08:59:02,963 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T08:59:02,964 DEBUG [RS:1;16bf49104b8f:37041 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512f8b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16bf49104b8f/172.17.0.2:0 2024-11-16T08:59:02,964 DEBUG [RS:0;16bf49104b8f:40505 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45a680cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16bf49104b8f/172.17.0.2:0 2024-11-16T08:59:02,978 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;16bf49104b8f:37041 2024-11-16T08:59:02,978 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;16bf49104b8f:40505 2024-11-16T08:59:02,981 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T08:59:02,981 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T08:59:02,981 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T08:59:02,981 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T08:59:02,981 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T08:59:02,981 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T08:59:02,984 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(2659): reportForDuty to master=16bf49104b8f,44307,1731747540960 with port=40505, startcode=1731747541750 2024-11-16T08:59:02,984 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(2659): reportForDuty to master=16bf49104b8f,44307,1731747540960 with port=37041, startcode=1731747541884 2024-11-16T08:59:02,995 DEBUG [RS:0;16bf49104b8f:40505 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T08:59:02,995 DEBUG [RS:1;16bf49104b8f:37041 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T08:59:03,021 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:03,034 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T08:59:03,044 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T08:59:03,051 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 16bf49104b8f,44307,1731747540960 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T08:59:03,059 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41697, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T08:59:03,059 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56033, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T08:59:03,060 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:03,060 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:03,060 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:03,061 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:03,061 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/16bf49104b8f:0, corePoolSize=10, maxPoolSize=10 2024-11-16T08:59:03,061 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,061 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/16bf49104b8f:0, corePoolSize=2, maxPoolSize=2 2024-11-16T08:59:03,061 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,065 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44307 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T08:59:03,067 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731747573067 2024-11-16T08:59:03,069 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T08:59:03,069 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:03,070 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T08:59:03,070 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T08:59:03,073 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44307 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-16T08:59:03,074 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,074 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T08:59:03,074 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T08:59:03,075 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T08:59:03,075 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T08:59:03,075 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T08:59:03,078 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,082 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T08:59:03,083 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T08:59:03,084 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T08:59:03,086 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T08:59:03,086 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T08:59:03,088 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.large.0-1731747543087,5,FailOnTimeoutGroup] 2024-11-16T08:59:03,089 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.small.0-1731747543088,5,FailOnTimeoutGroup] 2024-11-16T08:59:03,089 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,089 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T08:59:03,089 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T08:59:03,090 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:03,090 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,091 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,097 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:03,098 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T08:59:03,099 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T08:59:03,099 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-16T08:59:03,099 WARN [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T08:59:03,099 WARN [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-16T08:59:03,102 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T08:59:03,102 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,103 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,103 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T08:59:03,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T08:59:03,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,107 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T08:59:03,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T08:59:03,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T08:59:03,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T08:59:03,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,116 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,117 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T08:59:03,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:03,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:03,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T08:59:03,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T08:59:03,119 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T08:59:03,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T08:59:03,127 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T08:59:03,128 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71658011, jitterRate=0.06778757274150848}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T08:59:03,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731747543097Initializing all the Stores at 1731747543097Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747543097Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747543098 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747543098Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747543098Cleaning up temporary data from old regions at 1731747543118 (+20 ms)Region opened successfully at 1731747543131 (+13 ms) 2024-11-16T08:59:03,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T08:59:03,132 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T08:59:03,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T08:59:03,132 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T08:59:03,133 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T08:59:03,134 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T08:59:03,135 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731747543132Disabling compacts and flushes for region at 1731747543132Disabling writes for close at 1731747543132Writing region close event to WAL at 1731747543134 (+2 ms)Closed at 1731747543134 2024-11-16T08:59:03,137 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:03,138 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T08:59:03,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T08:59:03,152 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T08:59:03,154 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T08:59:03,201 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(2659): reportForDuty to master=16bf49104b8f,44307,1731747540960 with port=40505, startcode=1731747541750 2024-11-16T08:59:03,201 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(2659): reportForDuty to master=16bf49104b8f,44307,1731747540960 with port=37041, startcode=1731747541884 2024-11-16T08:59:03,203 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44307 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16bf49104b8f,40505,1731747541750 2024-11-16T08:59:03,205 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44307 {}] master.ServerManager(517): Registering regionserver=16bf49104b8f,40505,1731747541750 2024-11-16T08:59:03,212 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44307 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,212 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44307 {}] master.ServerManager(517): Registering regionserver=16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,212 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:03,212 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T08:59:03,213 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T08:59:03,214 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:03,214 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T08:59:03,214 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T08:59:03,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T08:59:03,272 DEBUG [RS:1;16bf49104b8f:37041 {}] zookeeper.ZKUtil(111): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,272 DEBUG [RS:0;16bf49104b8f:40505 {}] zookeeper.ZKUtil(111): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16bf49104b8f,40505,1731747541750 2024-11-16T08:59:03,272 WARN [RS:0;16bf49104b8f:40505 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T08:59:03,272 WARN [RS:1;16bf49104b8f:37041 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T08:59:03,273 INFO [RS:0;16bf49104b8f:40505 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:03,273 INFO [RS:1;16bf49104b8f:37041 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:03,273 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,273 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,40505,1731747541750 2024-11-16T08:59:03,275 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16bf49104b8f,37041,1731747541884] 2024-11-16T08:59:03,275 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16bf49104b8f,40505,1731747541750] 2024-11-16T08:59:03,301 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T08:59:03,301 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T08:59:03,305 WARN [16bf49104b8f:44307 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T08:59:03,323 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T08:59:03,323 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T08:59:03,327 INFO [RS:1;16bf49104b8f:37041 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T08:59:03,327 INFO [RS:0;16bf49104b8f:40505 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T08:59:03,327 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,327 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,328 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T08:59:03,328 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T08:59:03,332 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T08:59:03,332 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T08:59:03,334 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,334 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,334 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,334 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,334 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,334 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,334 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,334 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,334 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16bf49104b8f:0, corePoolSize=2, maxPoolSize=2 2024-11-16T08:59:03,335 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16bf49104b8f:0, corePoolSize=2, maxPoolSize=2 2024-11-16T08:59:03,335 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,335 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,336 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,336 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,336 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:03,336 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,336 DEBUG [RS:1;16bf49104b8f:37041 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:03,336 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,336 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:03,336 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:03,336 DEBUG [RS:0;16bf49104b8f:40505 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:03,337 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,337 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,338 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,40505,1731747541750-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T08:59:03,338 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,37041,1731747541884-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T08:59:03,364 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T08:59:03,364 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T08:59:03,366 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,40505,1731747541750-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,366 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,37041,1731747541884-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,366 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,366 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,367 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.Replication(171): 16bf49104b8f,40505,1731747541750 started 2024-11-16T08:59:03,367 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.Replication(171): 16bf49104b8f,37041,1731747541884 started 2024-11-16T08:59:03,392 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,392 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:03,393 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1482): Serving as 16bf49104b8f,40505,1731747541750, RpcServer on 16bf49104b8f/172.17.0.2:40505, sessionid=0x101442801920001 2024-11-16T08:59:03,393 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1482): Serving as 16bf49104b8f,37041,1731747541884, RpcServer on 16bf49104b8f/172.17.0.2:37041, sessionid=0x101442801920002 2024-11-16T08:59:03,394 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T08:59:03,394 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T08:59:03,394 DEBUG [RS:1;16bf49104b8f:37041 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,394 DEBUG [RS:0;16bf49104b8f:40505 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16bf49104b8f,40505,1731747541750 2024-11-16T08:59:03,394 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,37041,1731747541884' 2024-11-16T08:59:03,394 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,40505,1731747541750' 2024-11-16T08:59:03,394 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T08:59:03,394 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T08:59:03,395 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T08:59:03,395 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T08:59:03,396 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T08:59:03,396 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T08:59:03,396 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T08:59:03,396 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T08:59:03,396 DEBUG [RS:0;16bf49104b8f:40505 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16bf49104b8f,40505,1731747541750 2024-11-16T08:59:03,396 DEBUG [RS:1;16bf49104b8f:37041 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,397 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,40505,1731747541750' 2024-11-16T08:59:03,397 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,37041,1731747541884' 2024-11-16T08:59:03,397 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T08:59:03,397 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T08:59:03,397 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T08:59:03,397 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T08:59:03,398 DEBUG [RS:0;16bf49104b8f:40505 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T08:59:03,398 DEBUG [RS:1;16bf49104b8f:37041 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T08:59:03,398 INFO [RS:0;16bf49104b8f:40505 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T08:59:03,398 INFO [RS:1;16bf49104b8f:37041 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T08:59:03,398 INFO [RS:0;16bf49104b8f:40505 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T08:59:03,398 INFO [RS:1;16bf49104b8f:37041 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T08:59:03,507 INFO [RS:0;16bf49104b8f:40505 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T08:59:03,507 INFO [RS:1;16bf49104b8f:37041 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T08:59:03,508 INFO [RS:0;16bf49104b8f:40505 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16bf49104b8f%2C40505%2C1731747541750, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,40505,1731747541750, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs, maxLogs=55 2024-11-16T08:59:03,508 INFO [RS:1;16bf49104b8f:37041 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16bf49104b8f%2C37041%2C1731747541884, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,37041,1731747541884, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs, maxLogs=55 2024-11-16T08:59:03,534 INFO [RS:0;16bf49104b8f:40505 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,40505,1731747541750/16bf49104b8f%2C40505%2C1731747541750.1731747543511 2024-11-16T08:59:03,534 INFO [RS:1;16bf49104b8f:37041 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,37041,1731747541884/16bf49104b8f%2C37041%2C1731747541884.1731747543511 2024-11-16T08:59:03,534 DEBUG [RS:0;16bf49104b8f:40505 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:03,534 DEBUG [RS:1;16bf49104b8f:37041 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:03,557 DEBUG [16bf49104b8f:44307 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T08:59:03,569 DEBUG [16bf49104b8f:44307 {}] balancer.BalancerClusterState(204): Hosts are {16bf49104b8f=0} racks are {/default-rack=0} 2024-11-16T08:59:03,572 DEBUG [16bf49104b8f:44307 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T08:59:03,572 DEBUG [16bf49104b8f:44307 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T08:59:03,572 DEBUG [16bf49104b8f:44307 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T08:59:03,572 DEBUG [16bf49104b8f:44307 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T08:59:03,572 INFO [16bf49104b8f:44307 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T08:59:03,572 INFO [16bf49104b8f:44307 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T08:59:03,572 DEBUG [16bf49104b8f:44307 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T08:59:03,579 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,584 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16bf49104b8f,37041,1731747541884, state=OPENING 2024-11-16T08:59:03,620 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T08:59:03,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:03,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:03,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:03,630 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:03,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:03,631 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:03,633 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T08:59:03,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16bf49104b8f,37041,1731747541884}] 2024-11-16T08:59:03,819 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T08:59:03,822 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35839, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T08:59:03,830 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T08:59:03,831 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:03,832 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T08:59:03,834 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16bf49104b8f%2C37041%2C1731747541884.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,37041,1731747541884, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs, maxLogs=55 2024-11-16T08:59:03,854 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,37041,1731747541884/16bf49104b8f%2C37041%2C1731747541884.meta.1731747543836.meta 2024-11-16T08:59:03,855 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:03,855 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:03,857 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T08:59:03,860 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T08:59:03,865 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T08:59:03,869 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T08:59:03,870 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:03,870 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T08:59:03,870 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T08:59:03,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T08:59:03,873 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T08:59:03,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T08:59:03,874 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T08:59:03,874 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,875 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T08:59:03,876 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T08:59:03,876 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T08:59:03,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T08:59:03,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:03,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:03,877 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T08:59:03,879 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:03,880 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:03,880 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T08:59:03,880 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T08:59:03,881 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T08:59:03,882 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T08:59:03,882 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62853713, jitterRate=-0.06340669095516205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T08:59:03,883 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T08:59:03,885 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731747543871Writing region info on filesystem at 1731747543871Initializing all the Stores at 1731747543871Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747543871Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747543872 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747543872Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747543872Cleaning up temporary data from old regions at 1731747543880 (+8 ms)Running coprocessor post-open hooks at 1731747543883 (+3 ms)Region opened successfully at 1731747543884 (+1 ms) 2024-11-16T08:59:03,890 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731747543810 2024-11-16T08:59:03,899 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T08:59:03,900 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T08:59:03,900 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,902 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16bf49104b8f,37041,1731747541884, state=OPEN 2024-11-16T08:59:03,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T08:59:03,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T08:59:03,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T08:59:03,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:03,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:03,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:03,936 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16bf49104b8f,37041,1731747541884 2024-11-16T08:59:03,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T08:59:03,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16bf49104b8f,37041,1731747541884 in 299 msec 2024-11-16T08:59:03,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T08:59:03,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 799 msec 2024-11-16T08:59:03,947 DEBUG [PEWorker-4 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:03,948 INFO [PEWorker-4 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T08:59:03,965 DEBUG [PEWorker-4 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T08:59:03,966 DEBUG [PEWorker-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16bf49104b8f,37041,1731747541884, seqNum=-1] 2024-11-16T08:59:03,985 DEBUG [PEWorker-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T08:59:03,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34273, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T08:59:04,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0290 sec 2024-11-16T08:59:04,003 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731747544003, completionTime=-1 2024-11-16T08:59:04,006 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T08:59:04,006 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T08:59:04,032 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T08:59:04,032 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731747604032 2024-11-16T08:59:04,032 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731747664032 2024-11-16T08:59:04,032 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-16T08:59:04,034 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T08:59:04,040 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,44307,1731747540960-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:04,040 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,44307,1731747540960-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:04,040 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,44307,1731747540960-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:04,041 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-16bf49104b8f:44307, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:04,042 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:04,042 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:04,048 DEBUG [master/16bf49104b8f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T08:59:04,069 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.049sec 2024-11-16T08:59:04,071 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T08:59:04,072 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T08:59:04,073 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T08:59:04,073 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T08:59:04,073 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T08:59:04,074 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,44307,1731747540960-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T08:59:04,074 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,44307,1731747540960-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T08:59:04,076 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T08:59:04,076 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T08:59:04,077 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,44307,1731747540960-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:04,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T08:59:04,155 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-16T08:59:04,155 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-16T08:59:04,159 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16bf49104b8f,44307,-1 for getting cluster id 2024-11-16T08:59:04,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T08:59:04,168 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '00677e86-245f-40a3-8b82-c52b135b3607' 2024-11-16T08:59:04,171 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T08:59:04,171 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "00677e86-245f-40a3-8b82-c52b135b3607" 2024-11-16T08:59:04,173 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1f29a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T08:59:04,173 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16bf49104b8f,44307,-1] 2024-11-16T08:59:04,175 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T08:59:04,176 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:04,178 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T08:59:04,181 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27097cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T08:59:04,181 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T08:59:04,188 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16bf49104b8f,37041,1731747541884, seqNum=-1] 2024-11-16T08:59:04,188 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T08:59:04,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T08:59:04,219 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=16bf49104b8f,44307,1731747540960 2024-11-16T08:59:04,224 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T08:59:04,230 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 16bf49104b8f,44307,1731747540960 2024-11-16T08:59:04,232 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67a44e26 2024-11-16T08:59:04,236 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T08:59:04,240 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T08:59:04,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731747544219', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T08:59:04,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 2024-11-16T08:59:04,253 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T08:59:04,255 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731747544219" procId is: 4 2024-11-16T08:59:04,255 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:04,257 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T08:59:04,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:04,269 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 30bf084dd826fc23019168b4868ebb00, NAME => 'testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731747544219', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:04,275 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:04,275 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(1722): Closing 30bf084dd826fc23019168b4868ebb00, disabling compactions & flushes 2024-11-16T08:59:04,275 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:04,276 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:04,276 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. after waiting 0 ms 2024-11-16T08:59:04,276 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:04,276 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:04,276 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731747544219-pool-0 {}] regionserver.HRegion(1676): Region close journal for 30bf084dd826fc23019168b4868ebb00: Waiting for close lock at 1731747544275Disabling compacts and flushes for region at 1731747544275Disabling writes for close at 1731747544276 (+1 ms)Writing region close event to WAL at 1731747544276Closed at 1731747544276 2024-11-16T08:59:04,277 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T08:59:04,282 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731747544277"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731747544277"}]},"ts":"1731747544277"} 2024-11-16T08:59:04,286 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T08:59:04,287 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T08:59:04,289 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731747544219","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731747544287"}]},"ts":"1731747544287"} 2024-11-16T08:59:04,293 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731747544219, state=ENABLING in hbase:meta 2024-11-16T08:59:04,293 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(204): Hosts are {16bf49104b8f=0} racks are {/default-rack=0} 2024-11-16T08:59:04,294 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T08:59:04,294 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T08:59:04,294 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T08:59:04,294 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T08:59:04,294 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T08:59:04,294 INFO [PEWorker-2 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T08:59:04,294 DEBUG [PEWorker-2 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T08:59:04,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, ASSIGN}] 2024-11-16T08:59:04,297 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, ASSIGN 2024-11-16T08:59:04,298 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, ASSIGN; state=OFFLINE, location=16bf49104b8f,37041,1731747541884; forceNewPlan=false, retain=false 2024-11-16T08:59:04,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:04,454 INFO [16bf49104b8f:44307 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T08:59:04,456 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30bf084dd826fc23019168b4868ebb00, regionState=OPENING, regionLocation=16bf49104b8f,37041,1731747541884 2024-11-16T08:59:04,464 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, ASSIGN because future has completed 2024-11-16T08:59:04,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,37041,1731747541884}] 2024-11-16T08:59:04,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:04,620 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:04,621 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 30bf084dd826fc23019168b4868ebb00, NAME => 'testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:04,622 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731747544219 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,622 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:04,622 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,622 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,624 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,627 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30bf084dd826fc23019168b4868ebb00 columnFamilyName family 2024-11-16T08:59:04,628 DEBUG [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:04,628 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.HStore(327): Store=30bf084dd826fc23019168b4868ebb00/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:04,628 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,629 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,629 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,630 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,630 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,630 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,636 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T08:59:04,636 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 30bf084dd826fc23019168b4868ebb00; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73574450, jitterRate=0.09634473919868469}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T08:59:04,636 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:04,637 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 30bf084dd826fc23019168b4868ebb00: Running coprocessor pre-open hook at 1731747544623Writing region info on filesystem at 1731747544623Initializing all the Stores at 1731747544623Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747544623Cleaning up temporary data from old regions at 1731747544630 (+7 ms)Running coprocessor post-open hooks at 1731747544636 (+6 ms)Region opened successfully at 1731747544637 (+1 ms) 2024-11-16T08:59:04,638 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00., pid=6, masterSystemTime=1731747544617 2024-11-16T08:59:04,641 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:04,641 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:04,642 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=30bf084dd826fc23019168b4868ebb00, regionState=OPEN, openSeqNum=2, regionLocation=16bf49104b8f,37041,1731747541884 2024-11-16T08:59:04,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,37041,1731747541884 because future has completed 2024-11-16T08:59:04,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T08:59:04,648 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,37041,1731747541884 in 181 msec 2024-11-16T08:59:04,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T08:59:04,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, ASSIGN in 353 msec 2024-11-16T08:59:04,651 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T08:59:04,652 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731747544219","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731747544651"}]},"ts":"1731747544651"} 2024-11-16T08:59:04,654 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731747544219, state=ENABLED in hbase:meta 2024-11-16T08:59:04,655 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T08:59:04,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731747544219 in 408 msec 2024-11-16T08:59:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:04,888 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731747544219 completed 2024-11-16T08:59:04,888 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731747544219 get assigned. Timeout = 60000ms 2024-11-16T08:59:04,894 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T08:59:04,900 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731747544219 assigned to meta. Checking AM states. 2024-11-16T08:59:04,900 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T08:59:04,901 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731747544219 assigned. 2024-11-16T08:59:04,908 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731747544219', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00., hostname=16bf49104b8f,37041,1731747541884, seqNum=2] 2024-11-16T08:59:04,919 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 30bf084dd826fc23019168b4868ebb00 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T08:59:04,940 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 46fc8315bd524c998c23540ef2d78ff1 2024-11-16T08:59:04,974 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/.tmp/family/46fc8315bd524c998c23540ef2d78ff1 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T08:59:04,978 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/.tmp/family/46fc8315bd524c998c23540ef2d78ff1 2024-11-16T08:59:05,011 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/.tmp/family/46fc8315bd524c998c23540ef2d78ff1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1 2024-11-16T08:59:05,012 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1, delay=901 ms 2024-11-16T08:59:05,014 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T08:59:05,020 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 30bf084dd826fc23019168b4868ebb00 in 100ms, sequenceid=6, compaction requested=false 2024-11-16T08:59:05,021 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731747544219' 2024-11-16T08:59:05,023 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 30bf084dd826fc23019168b4868ebb00: 2024-11-16T08:59:05,919 DEBUG [hfile-prefetch-1731747545012 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46fc8315bd524c998c23540ef2d78ff1_0. 2024-11-16T08:59:05,927 DEBUG [hfile-prefetch-1731747545012 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 46fc8315bd524c998c23540ef2d78ff1_117. 2024-11-16T08:59:05,928 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1, with total blocks 2, and data blocks 1 2024-11-16T08:59:05,928 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T08:59:05,929 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T08:59:05,929 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1. Will sleep 100ms and try the verification again. 2024-11-16T08:59:06,028 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T08:59:06,028 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T08:59:06,028 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:06,028 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:06,029 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1, with total blocks 2, and data blocks 1 2024-11-16T08:59:06,029 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2285): iterating over 1 entries in the backing map 2024-11-16T08:59:06,029 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2293): found block for file 46fc8315bd524c998c23540ef2d78ff1 in the backing map. Acquiring read lock for offset 0 2024-11-16T08:59:06,030 DEBUG [hfile-prefetch-1731747545012 {}] bucket.BucketCache(2305): File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1 has now been fully cached. 2024-11-16T08:59:06,030 DEBUG [hfile-prefetch-1731747545012 {}] hfile.PrefetchExecutor(121): Prefetch completed for 46fc8315bd524c998c23540ef2d78ff1 2024-11-16T08:59:06,035 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=30bf084dd826fc23019168b4868ebb00, source=16bf49104b8f,37041,1731747541884, destination=16bf49104b8f,40505,1731747541750, warming up region on 16bf49104b8f,40505,1731747541750 2024-11-16T08:59:06,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T08:59:06,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=30bf084dd826fc23019168b4868ebb00, source=16bf49104b8f,37041,1731747541884, destination=16bf49104b8f,40505,1731747541750, running balancer 2024-11-16T08:59:06,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, REOPEN/MOVE 2024-11-16T08:59:06,039 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, REOPEN/MOVE 2024-11-16T08:59:06,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45119, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T08:59:06,041 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(7855): Warmup {ENCODED => 30bf084dd826fc23019168b4868ebb00, NAME => 'testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:06,041 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=30bf084dd826fc23019168b4868ebb00, regionState=CLOSING, regionLocation=16bf49104b8f,37041,1731747541884 2024-11-16T08:59:06,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:06,042 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,042 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30bf084dd826fc23019168b4868ebb00 columnFamilyName family 2024-11-16T08:59:06,042 DEBUG [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:06,044 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T08:59:06,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, REOPEN/MOVE because future has completed 2024-11-16T08:59:06,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T08:59:06,044 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:06,045 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:06,045 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-16T08:59:06,045 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,37041,1731747541884}] 2024-11-16T08:59:06,046 DEBUG [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1 2024-11-16T08:59:06,046 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.HStore(327): Store=30bf084dd826fc23019168b4868ebb00/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(1722): Closing 30bf084dd826fc23019168b4868ebb00, disabling compactions & flushes 2024-11-16T08:59:06,047 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. after waiting 0 ms 2024-11-16T08:59:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,048 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:06,048 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747546044 2024-11-16T08:59:06,048 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T08:59:06,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40505 {}] regionserver.HRegion(1676): Region close journal for 30bf084dd826fc23019168b4868ebb00: Waiting for close lock at 1731747546047Disabling compacts and flushes for region at 1731747546047Disabling writes for close at 1731747546047Writing region close event to WAL at 1731747546048 (+1 ms)Closed at 1731747546048 2024-11-16T08:59:06,050 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 1 2024-11-16T08:59:06,050 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 1, file name: bucket.persistence1731747546028 2024-11-16T08:59:06,050 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T08:59:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44307 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-16T08:59:06,206 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,206 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-16T08:59:06,207 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 30bf084dd826fc23019168b4868ebb00, disabling compactions & flushes 2024-11-16T08:59:06,207 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,208 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,208 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. after waiting 0 ms 2024-11-16T08:59:06,208 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,210 DEBUG [StoreFileCloser-30bf084dd826fc23019168b4868ebb00-family-1 {}] bucket.BucketCache(768): Removing file 46fc8315bd524c998c23540ef2d78ff1 for region 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,211 DEBUG [StoreFileCloser-30bf084dd826fc23019168b4868ebb00-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 46fc8315bd524c998c23540ef2d78ff1, starting offset: 0, end offset: 9223372036854775807 2024-11-16T08:59:06,213 DEBUG [StoreFileCloser-30bf084dd826fc23019168b4868ebb00-family-1 {}] bucket.BucketCache(856): removed key 46fc8315bd524c998c23540ef2d78ff1_0 from back map with offset lock 2094080 in the evict process 2024-11-16T08:59:06,220 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T08:59:06,221 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,221 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 30bf084dd826fc23019168b4868ebb00: Waiting for close lock at 1731747546207Running coprocessor pre-close hooks at 1731747546207Disabling compacts and flushes for region at 1731747546207Disabling writes for close at 1731747546208 (+1 ms)Writing region close event to WAL at 1731747546214 (+6 ms)Running coprocessor post-close hooks at 1731747546220 (+6 ms)Closed at 1731747546221 (+1 ms) 2024-11-16T08:59:06,222 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 30bf084dd826fc23019168b4868ebb00 move to 16bf49104b8f,40505,1731747541750 record at close sequenceid=6 2024-11-16T08:59:06,225 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,226 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=30bf084dd826fc23019168b4868ebb00, regionState=CLOSED 2024-11-16T08:59:06,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,37041,1731747541884 because future has completed 2024-11-16T08:59:06,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T08:59:06,234 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,37041,1731747541884 in 186 msec 2024-11-16T08:59:06,235 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, REOPEN/MOVE; state=CLOSED, location=16bf49104b8f,40505,1731747541750; forceNewPlan=false, retain=false 2024-11-16T08:59:06,249 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T08:59:06,249 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T08:59:06,249 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:06,249 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:06,251 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T08:59:06,251 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T08:59:06,251 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:06,251 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:06,252 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:06,252 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747546249 2024-11-16T08:59:06,252 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T08:59:06,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:06,254 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747546251 2024-11-16T08:59:06,255 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T08:59:06,386 INFO [16bf49104b8f:44307 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T08:59:06,387 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=30bf084dd826fc23019168b4868ebb00, regionState=OPENING, regionLocation=16bf49104b8f,40505,1731747541750 2024-11-16T08:59:06,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, REOPEN/MOVE because future has completed 2024-11-16T08:59:06,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,40505,1731747541750}] 2024-11-16T08:59:06,549 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,549 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 30bf084dd826fc23019168b4868ebb00, NAME => 'testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:06,550 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731747544219 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,550 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:06,550 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,550 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,552 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,553 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 30bf084dd826fc23019168b4868ebb00 columnFamilyName family 2024-11-16T08:59:06,553 DEBUG [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:06,556 DEBUG [StoreFileOpener-30bf084dd826fc23019168b4868ebb00-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1, delay=939 ms 2024-11-16T08:59:06,557 DEBUG [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1 2024-11-16T08:59:06,558 INFO [StoreOpener-30bf084dd826fc23019168b4868ebb00-1 {}] regionserver.HStore(327): Store=30bf084dd826fc23019168b4868ebb00/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:06,558 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,559 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,560 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,560 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,560 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,566 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,567 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 30bf084dd826fc23019168b4868ebb00; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75308387, jitterRate=0.12218241393566132}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T08:59:06,568 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,569 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 30bf084dd826fc23019168b4868ebb00: Running coprocessor pre-open hook at 1731747546551Writing region info on filesystem at 1731747546551Initializing all the Stores at 1731747546551Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747546551Cleaning up temporary data from old regions at 1731747546560 (+9 ms)Running coprocessor post-open hooks at 1731747546568 (+8 ms)Region opened successfully at 1731747546568 2024-11-16T08:59:06,569 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00., pid=9, masterSystemTime=1731747546545 2024-11-16T08:59:06,573 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,573 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,574 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=30bf084dd826fc23019168b4868ebb00, regionState=OPEN, openSeqNum=10, regionLocation=16bf49104b8f,40505,1731747541750 2024-11-16T08:59:06,576 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,40505,1731747541750 because future has completed 2024-11-16T08:59:06,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-16T08:59:06,579 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 30bf084dd826fc23019168b4868ebb00, server=16bf49104b8f,40505,1731747541750 in 185 msec 2024-11-16T08:59:06,581 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731747544219, region=30bf084dd826fc23019168b4868ebb00, REOPEN/MOVE in 542 msec 2024-11-16T08:59:06,592 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T08:59:06,592 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T08:59:06,592 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:06,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:06,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:06,597 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T08:59:06,597 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T08:59:06,597 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1694793401, stopped=false 2024-11-16T08:59:06,598 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=16bf49104b8f,44307,1731747540960 2024-11-16T08:59:06,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:06,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:06,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:06,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:06,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:06,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:06,637 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T08:59:06,637 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T08:59:06,638 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:06,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:06,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:06,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:06,639 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16bf49104b8f,40505,1731747541750' ***** 2024-11-16T08:59:06,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:06,640 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T08:59:06,640 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16bf49104b8f,37041,1731747541884' ***** 2024-11-16T08:59:06,640 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T08:59:06,640 INFO [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T08:59:06,641 INFO [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T08:59:06,641 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T08:59:06,641 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T08:59:06,641 INFO [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.persistence 2024-11-16T08:59:06,641 INFO [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/bucket.persistence 2024-11-16T08:59:06,641 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T08:59:06,641 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T08:59:06,641 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T08:59:06,642 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T08:59:06,642 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T08:59:06,642 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T08:59:06,642 INFO [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T08:59:06,642 INFO [regionserver/16bf49104b8f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:06,642 INFO [regionserver/16bf49104b8f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:06,643 INFO [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T08:59:06,643 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T08:59:06,643 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T08:59:06,653 DEBUG [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1487): Thread RS:1;16bf49104b8f:37041 started persisting bucket cache to file 2024-11-16T08:59:06,653 DEBUG [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:06,653 DEBUG [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:06,653 DEBUG [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1487): Thread RS:0;16bf49104b8f:40505 started persisting bucket cache to file 2024-11-16T08:59:06,653 DEBUG [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:06,653 DEBUG [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:06,656 DEBUG [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:06,656 DEBUG [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:06,656 DEBUG [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747546653 2024-11-16T08:59:06,656 DEBUG [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747546653 2024-11-16T08:59:06,656 DEBUG [RS:1;16bf49104b8f:37041 {}] bucket.BucketCache(1509): Thread RS:1;16bf49104b8f:37041 finished persisting bucket cache to file, renaming 2024-11-16T08:59:06,656 DEBUG [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1509): Thread RS:0;16bf49104b8f:40505 finished persisting bucket cache to file, renaming 2024-11-16T08:59:06,656 WARN [RS:0;16bf49104b8f:40505 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-16T08:59:06,656 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T08:59:06,656 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T08:59:06,656 INFO [RS:1;16bf49104b8f:37041 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T08:59:06,656 INFO [RS:0;16bf49104b8f:40505 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T08:59:06,657 INFO [RS:0;16bf49104b8f:40505 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T08:59:06,657 INFO [RS:1;16bf49104b8f:37041 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T08:59:06,657 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(959): stopping server 16bf49104b8f,37041,1731747541884 2024-11-16T08:59:06,657 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(3091): Received CLOSE for 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,657 INFO [RS:1;16bf49104b8f:37041 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T08:59:06,657 INFO [RS:1;16bf49104b8f:37041 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;16bf49104b8f:37041. 2024-11-16T08:59:06,657 DEBUG [RS:1;16bf49104b8f:37041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:06,657 DEBUG [RS:1;16bf49104b8f:37041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:06,657 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(959): stopping server 16bf49104b8f,40505,1731747541750 2024-11-16T08:59:06,657 INFO [RS:0;16bf49104b8f:40505 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T08:59:06,657 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T08:59:06,657 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T08:59:06,657 INFO [RS:0;16bf49104b8f:40505 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;16bf49104b8f:40505. 2024-11-16T08:59:06,657 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T08:59:06,658 DEBUG [RS:0;16bf49104b8f:40505 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:06,658 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T08:59:06,658 DEBUG [RS:0;16bf49104b8f:40505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:06,658 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T08:59:06,658 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T08:59:06,658 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T08:59:06,658 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1325): Online Regions={30bf084dd826fc23019168b4868ebb00=testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00.} 2024-11-16T08:59:06,658 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-16T08:59:06,658 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 30bf084dd826fc23019168b4868ebb00, disabling compactions & flushes 2024-11-16T08:59:06,658 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,658 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,658 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. after waiting 0 ms 2024-11-16T08:59:06,658 DEBUG [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-16T08:59:06,658 DEBUG [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1351): Waiting on 30bf084dd826fc23019168b4868ebb00 2024-11-16T08:59:06,658 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,659 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T08:59:06,659 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T08:59:06,659 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T08:59:06,659 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T08:59:06,659 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T08:59:06,659 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T08:59:06,659 DEBUG [StoreFileCloser-30bf084dd826fc23019168b4868ebb00-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/family/46fc8315bd524c998c23540ef2d78ff1 2024-11-16T08:59:06,659 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-16T08:59:06,663 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnRegionMove1731747544219/30bf084dd826fc23019168b4868ebb00/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T08:59:06,663 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,664 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 30bf084dd826fc23019168b4868ebb00: Waiting for close lock at 1731747546657Running coprocessor pre-close hooks at 1731747546658 (+1 ms)Disabling compacts and flushes for region at 1731747546658Disabling writes for close at 1731747546658Writing region close event to WAL at 1731747546659 (+1 ms)Running coprocessor post-close hooks at 1731747546663 (+4 ms)Closed at 1731747546663 2024-11-16T08:59:06,664 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00. 2024-11-16T08:59:06,668 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c732dfe4b3c74e6f847f8ce8dbb664c3 2024-11-16T08:59:06,687 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/info/c732dfe4b3c74e6f847f8ce8dbb664c3 is 205, key is testBlockEvictionOnRegionMove1731747544219,,1731747544241.30bf084dd826fc23019168b4868ebb00./info:regioninfo/1731747546573/Put/seqid=0 2024-11-16T08:59:06,688 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/info/c732dfe4b3c74e6f847f8ce8dbb664c3 2024-11-16T08:59:06,697 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on edab83b19b1146238e6f6bc91dd459fc 2024-11-16T08:59:06,710 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/ns/edab83b19b1146238e6f6bc91dd459fc is 43, key is default/ns:d/1731747543990/Put/seqid=0 2024-11-16T08:59:06,710 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/ns/edab83b19b1146238e6f6bc91dd459fc 2024-11-16T08:59:06,721 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 5c0982371122417390bac557869fda0f 2024-11-16T08:59:06,733 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/table/5c0982371122417390bac557869fda0f is 78, key is testBlockEvictionOnRegionMove1731747544219/table:state/1731747544651/Put/seqid=0 2024-11-16T08:59:06,733 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/table/5c0982371122417390bac557869fda0f 2024-11-16T08:59:06,736 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/info/c732dfe4b3c74e6f847f8ce8dbb664c3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/c732dfe4b3c74e6f847f8ce8dbb664c3 2024-11-16T08:59:06,737 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/c732dfe4b3c74e6f847f8ce8dbb664c3, delay=984 ms 2024-11-16T08:59:06,739 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/c732dfe4b3c74e6f847f8ce8dbb664c3, entries=15, sequenceid=15, filesize=7.6 K 2024-11-16T08:59:06,739 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/ns/edab83b19b1146238e6f6bc91dd459fc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/edab83b19b1146238e6f6bc91dd459fc 2024-11-16T08:59:06,740 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/edab83b19b1146238e6f6bc91dd459fc, delay=935 ms 2024-11-16T08:59:06,742 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/edab83b19b1146238e6f6bc91dd459fc, entries=2, sequenceid=15, filesize=5.0 K 2024-11-16T08:59:06,742 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/table/5c0982371122417390bac557869fda0f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/5c0982371122417390bac557869fda0f 2024-11-16T08:59:06,743 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/5c0982371122417390bac557869fda0f, delay=977 ms 2024-11-16T08:59:06,744 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/5c0982371122417390bac557869fda0f, entries=2, sequenceid=15, filesize=5.3 K 2024-11-16T08:59:06,745 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 86ms, sequenceid=15, compaction requested=false 2024-11-16T08:59:06,745 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-16T08:59:06,746 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/c732dfe4b3c74e6f847f8ce8dbb664c3 2024-11-16T08:59:06,749 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/edab83b19b1146238e6f6bc91dd459fc 2024-11-16T08:59:06,749 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/5c0982371122417390bac557869fda0f 2024-11-16T08:59:06,753 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-16T08:59:06,754 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T08:59:06,754 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T08:59:06,754 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731747546659Running coprocessor pre-close hooks at 1731747546659Disabling compacts and flushes for region at 1731747546659Disabling writes for close at 1731747546659Obtaining lock to block concurrent updates at 1731747546659Preparing flush snapshotting stores in 1588230740 at 1731747546659Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731747546660 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731747546660Flushing 1588230740/info: creating writer at 1731747546660Flushing 1588230740/info: appending metadata at 1731747546686 (+26 ms)Flushing 1588230740/info: closing flushed file at 1731747546686Flushing 1588230740/ns: creating writer at 1731747546691 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731747546709 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731747546709Flushing 1588230740/table: creating writer at 1731747546713 (+4 ms)Flushing 1588230740/table: appending metadata at 1731747546732 (+19 ms)Flushing 1588230740/table: closing flushed file at 1731747546732Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5399e21: reopening flushed file at 1731747546736 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e419e2f: reopening flushed file at 1731747546739 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a9e5f9a: reopening flushed file at 1731747546742 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 86ms, sequenceid=15, compaction requested=false at 1731747546745 (+3 ms)Writing region close event to WAL at 1731747546750 (+5 ms)Running coprocessor post-close hooks at 1731747546753 (+3 ms)Closed at 1731747546754 (+1 ms) 2024-11-16T08:59:06,754 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T08:59:06,859 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(976): stopping server 16bf49104b8f,37041,1731747541884; all regions closed. 2024-11-16T08:59:06,859 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(976): stopping server 16bf49104b8f,40505,1731747541750; all regions closed. 2024-11-16T08:59:06,867 DEBUG [RS:1;16bf49104b8f:37041 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs 2024-11-16T08:59:06,867 DEBUG [RS:0;16bf49104b8f:40505 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs 2024-11-16T08:59:06,867 INFO [RS:0;16bf49104b8f:40505 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16bf49104b8f%2C40505%2C1731747541750:(num 1731747543511) 2024-11-16T08:59:06,867 INFO [RS:1;16bf49104b8f:37041 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16bf49104b8f%2C37041%2C1731747541884.meta:.meta(num 1731747543836) 2024-11-16T08:59:06,867 DEBUG [RS:0;16bf49104b8f:40505 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:06,868 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:06,868 INFO [RS:0;16bf49104b8f:40505 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T08:59:06,868 INFO [RS:0;16bf49104b8f:40505 {}] hbase.ChoreService(370): Chore service for: regionserver/16bf49104b8f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T08:59:06,869 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T08:59:06,869 INFO [regionserver/16bf49104b8f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T08:59:06,869 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T08:59:06,869 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T08:59:06,869 INFO [RS:0;16bf49104b8f:40505 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T08:59:06,871 DEBUG [RS:1;16bf49104b8f:37041 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs 2024-11-16T08:59:06,871 INFO [RS:1;16bf49104b8f:37041 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16bf49104b8f%2C37041%2C1731747541884:(num 1731747543511) 2024-11-16T08:59:06,871 DEBUG [RS:1;16bf49104b8f:37041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:06,871 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:06,871 INFO [RS:0;16bf49104b8f:40505 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40505 2024-11-16T08:59:06,871 INFO [RS:1;16bf49104b8f:37041 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T08:59:06,872 INFO [RS:1;16bf49104b8f:37041 {}] hbase.ChoreService(370): Chore service for: regionserver/16bf49104b8f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T08:59:06,872 INFO [RS:1;16bf49104b8f:37041 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T08:59:06,872 INFO [regionserver/16bf49104b8f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T08:59:06,873 INFO [RS:1;16bf49104b8f:37041 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37041 2024-11-16T08:59:06,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16bf49104b8f,40505,1731747541750 2024-11-16T08:59:06,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T08:59:06,903 INFO [RS:0;16bf49104b8f:40505 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T08:59:06,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16bf49104b8f,37041,1731747541884 2024-11-16T08:59:06,911 INFO [RS:1;16bf49104b8f:37041 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T08:59:06,911 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16bf49104b8f,37041,1731747541884] 2024-11-16T08:59:06,927 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16bf49104b8f,37041,1731747541884 already deleted, retry=false 2024-11-16T08:59:06,928 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16bf49104b8f,37041,1731747541884 expired; onlineServers=1 2024-11-16T08:59:06,928 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16bf49104b8f,40505,1731747541750] 2024-11-16T08:59:06,936 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16bf49104b8f,40505,1731747541750 already deleted, retry=false 2024-11-16T08:59:06,936 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16bf49104b8f,40505,1731747541750 expired; onlineServers=0 2024-11-16T08:59:06,936 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '16bf49104b8f,44307,1731747540960' ***** 2024-11-16T08:59:06,936 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T08:59:06,936 INFO [M:0;16bf49104b8f:44307 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T08:59:06,936 INFO [M:0;16bf49104b8f:44307 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T08:59:06,936 DEBUG [M:0;16bf49104b8f:44307 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T08:59:06,936 DEBUG [M:0;16bf49104b8f:44307 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T08:59:06,936 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T08:59:06,936 DEBUG [master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.small.0-1731747543088 {}] cleaner.HFileCleaner(306): Exit Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.small.0-1731747543088,5,FailOnTimeoutGroup] 2024-11-16T08:59:06,936 DEBUG [master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.large.0-1731747543087 {}] cleaner.HFileCleaner(306): Exit Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.large.0-1731747543087,5,FailOnTimeoutGroup] 2024-11-16T08:59:06,937 INFO [M:0;16bf49104b8f:44307 {}] hbase.ChoreService(370): Chore service for: master/16bf49104b8f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T08:59:06,937 INFO [M:0;16bf49104b8f:44307 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T08:59:06,937 DEBUG [M:0;16bf49104b8f:44307 {}] master.HMaster(1795): Stopping service threads 2024-11-16T08:59:06,937 INFO [M:0;16bf49104b8f:44307 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T08:59:06,937 INFO [M:0;16bf49104b8f:44307 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T08:59:06,938 INFO [M:0;16bf49104b8f:44307 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T08:59:06,938 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T08:59:06,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T08:59:06,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:06,945 DEBUG [M:0;16bf49104b8f:44307 {}] zookeeper.ZKUtil(347): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T08:59:06,945 WARN [M:0;16bf49104b8f:44307 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T08:59:06,945 INFO [M:0;16bf49104b8f:44307 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.lastflushedseqids 2024-11-16T08:59:06,953 INFO [M:0;16bf49104b8f:44307 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T08:59:06,954 INFO [M:0;16bf49104b8f:44307 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T08:59:06,954 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T08:59:06,954 INFO [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:06,954 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:06,954 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T08:59:06,954 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:06,954 INFO [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.69 KB heapSize=43.02 KB 2024-11-16T08:59:06,963 DEBUG [M:0;16bf49104b8f:44307 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9fb512da6244425ba9526be29f59a1fd 2024-11-16T08:59:06,978 DEBUG [M:0;16bf49104b8f:44307 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9fb512da6244425ba9526be29f59a1fd is 82, key is hbase:meta,,1/info:regioninfo/1731747543900/Put/seqid=0 2024-11-16T08:59:06,979 INFO [M:0;16bf49104b8f:44307 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9fb512da6244425ba9526be29f59a1fd 2024-11-16T08:59:06,986 DEBUG [M:0;16bf49104b8f:44307 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c40b9dfa33c64b9ba981c74d864de3c4 2024-11-16T08:59:06,998 DEBUG [M:0;16bf49104b8f:44307 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c40b9dfa33c64b9ba981c74d864de3c4 is 779, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731747544657/Put/seqid=0 2024-11-16T08:59:06,999 INFO [M:0;16bf49104b8f:44307 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.07 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c40b9dfa33c64b9ba981c74d864de3c4 2024-11-16T08:59:07,006 DEBUG [M:0;16bf49104b8f:44307 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9f0a7eb1dc59408da4c333f1f1f1d5af 2024-11-16T08:59:07,018 DEBUG [M:0;16bf49104b8f:44307 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9f0a7eb1dc59408da4c333f1f1f1d5af is 69, key is 16bf49104b8f,37041,1731747541884/rs:state/1731747543212/Put/seqid=0 2024-11-16T08:59:07,019 INFO [M:0;16bf49104b8f:44307 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9f0a7eb1dc59408da4c333f1f1f1d5af 2024-11-16T08:59:07,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:07,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40505-0x101442801920001, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:07,020 INFO [RS:0;16bf49104b8f:40505 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T08:59:07,020 INFO [RS:0;16bf49104b8f:40505 {}] regionserver.HRegionServer(1031): Exiting; stopping=16bf49104b8f,40505,1731747541750; zookeeper connection closed. 2024-11-16T08:59:07,021 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@485bd3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@485bd3 2024-11-16T08:59:07,021 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9fb512da6244425ba9526be29f59a1fd as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9fb512da6244425ba9526be29f59a1fd 2024-11-16T08:59:07,022 INFO [M:0;16bf49104b8f:44307 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9fb512da6244425ba9526be29f59a1fd, entries=8, sequenceid=84, filesize=5.5 K 2024-11-16T08:59:07,023 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c40b9dfa33c64b9ba981c74d864de3c4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c40b9dfa33c64b9ba981c74d864de3c4 2024-11-16T08:59:07,024 INFO [M:0;16bf49104b8f:44307 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c40b9dfa33c64b9ba981c74d864de3c4, entries=9, sequenceid=84, filesize=6.5 K 2024-11-16T08:59:07,025 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9f0a7eb1dc59408da4c333f1f1f1d5af as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9f0a7eb1dc59408da4c333f1f1f1d5af 2024-11-16T08:59:07,026 INFO [M:0;16bf49104b8f:44307 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9f0a7eb1dc59408da4c333f1f1f1d5af, entries=2, sequenceid=84, filesize=5.1 K 2024-11-16T08:59:07,027 INFO [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false 2024-11-16T08:59:07,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:07,028 INFO [RS:1;16bf49104b8f:37041 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T08:59:07,028 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37041-0x101442801920002, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:07,028 INFO [RS:1;16bf49104b8f:37041 {}] regionserver.HRegionServer(1031): Exiting; stopping=16bf49104b8f,37041,1731747541884; zookeeper connection closed. 2024-11-16T08:59:07,028 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@59a750f2 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@59a750f2 2024-11-16T08:59:07,029 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-16T08:59:07,030 INFO [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:07,030 DEBUG [M:0;16bf49104b8f:44307 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731747546954Disabling compacts and flushes for region at 1731747546954Disabling writes for close at 1731747546954Obtaining lock to block concurrent updates at 1731747546954Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731747546954Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35518, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731747546955 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731747546955Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731747546955Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731747546978 (+23 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731747546978Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731747546981 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731747546998 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731747546998Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731747547001 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731747547018 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731747547018Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21955de2: reopening flushed file at 1731747547020 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a11bb59: reopening flushed file at 1731747547023 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@277f6ec4: reopening flushed file at 1731747547025 (+2 ms)Finished flush of dataSize ~34.69 KB/35518, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 73ms, sequenceid=84, compaction requested=false at 1731747547028 (+3 ms)Writing region close event to WAL at 1731747547029 (+1 ms)Closed at 1731747547029 2024-11-16T08:59:07,032 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T08:59:07,032 INFO [M:0;16bf49104b8f:44307 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T08:59:07,032 INFO [M:0;16bf49104b8f:44307 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44307 2024-11-16T08:59:07,032 INFO [M:0;16bf49104b8f:44307 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T08:59:07,159 INFO [M:0;16bf49104b8f:44307 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T08:59:07,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:07,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44307-0x101442801920000, quorum=127.0.0.1:51196, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:07,165 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T08:59:07,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T08:59:07,187 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,194 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/16bf49104b8f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/16bf49104b8f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731747546737 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731747545012 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/16bf49104b8f:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731747546740 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731747546556 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/16bf49104b8f:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=140 (was 100) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6613 (was 6961) 2024-11-16T08:59:07,199 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=140, ProcessCount=11, AvailableMemoryMB=6613 2024-11-16T08:59:07,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/hadoop-log-dir so I do NOT create it in target/test-data/00618538-4190-b3db-f3cc-279b10655cca 2024-11-16T08:59:07,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/hadoop-tmp-dir so I do NOT create it in target/test-data/00618538-4190-b3db-f3cc-279b10655cca 2024-11-16T08:59:07,200 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,202 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/cluster_8ad7f8c1-9ed3-e691-5cb6-923552e376a4, deleteOnExit=true 2024-11-16T08:59:07,205 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/cluster_8ad7f8c1-9ed3-e691-5cb6-923552e376a4/zookeeper_0, clientPort=61244, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/cluster_8ad7f8c1-9ed3-e691-5cb6-923552e376a4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/cluster_8ad7f8c1-9ed3-e691-5cb6-923552e376a4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-16T08:59:07,206 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61244 2024-11-16T08:59:07,217 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 with version=8 2024-11-16T08:59:07,217 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/hbase-staging 2024-11-16T08:59:07,221 INFO [Time-limited test {}] client.ConnectionUtils(128): master/16bf49104b8f:0 server-side Connection retries=45 2024-11-16T08:59:07,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,221 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T08:59:07,221 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,222 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T08:59:07,222 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-16T08:59:07,222 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T08:59:07,223 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35741 2024-11-16T08:59:07,225 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35741 connecting to ZooKeeper ensemble=127.0.0.1:61244 2024-11-16T08:59:07,264 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:357410x0, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T08:59:07,265 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35741-0x10144281b0a0000 connected 2024-11-16T08:59:07,362 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,362 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,365 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:07,366 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8, hbase.cluster.distributed=false 2024-11-16T08:59:07,371 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T08:59:07,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35741 2024-11-16T08:59:07,372 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35741 2024-11-16T08:59:07,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35741 2024-11-16T08:59:07,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35741 2024-11-16T08:59:07,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35741 2024-11-16T08:59:07,393 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16bf49104b8f:0 server-side Connection retries=45 2024-11-16T08:59:07,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,393 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T08:59:07,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,393 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T08:59:07,393 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T08:59:07,393 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T08:59:07,394 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42025 2024-11-16T08:59:07,396 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42025 connecting to ZooKeeper ensemble=127.0.0.1:61244 2024-11-16T08:59:07,396 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,396 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:420250x0, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T08:59:07,408 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42025-0x10144281b0a0001 connected 2024-11-16T08:59:07,408 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:07,408 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T08:59:07,409 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.cache 2024-11-16T08:59:07,409 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T08:59:07,409 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T08:59:07,409 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T08:59:07,409 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T08:59:07,409 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T08:59:07,409 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T08:59:07,410 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T08:59:07,410 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T08:59:07,410 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence is complete. 2024-11-16T08:59:07,411 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T08:59:07,412 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T08:59:07,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42025 2024-11-16T08:59:07,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42025 2024-11-16T08:59:07,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42025 2024-11-16T08:59:07,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42025 2024-11-16T08:59:07,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42025 2024-11-16T08:59:07,432 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16bf49104b8f:0 server-side Connection retries=45 2024-11-16T08:59:07,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,432 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T08:59:07,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:07,432 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T08:59:07,432 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T08:59:07,433 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T08:59:07,433 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38779 2024-11-16T08:59:07,435 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38779 connecting to ZooKeeper ensemble=127.0.0.1:61244 2024-11-16T08:59:07,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:387790x0, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T08:59:07,445 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38779-0x10144281b0a0002 connected 2024-11-16T08:59:07,445 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:07,445 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T08:59:07,446 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.cache 2024-11-16T08:59:07,446 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T08:59:07,446 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T08:59:07,446 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T08:59:07,446 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T08:59:07,446 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-16T08:59:07,446 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T08:59:07,447 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T08:59:07,447 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T08:59:07,447 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence is complete. 2024-11-16T08:59:07,447 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-16T08:59:07,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T08:59:07,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38779 2024-11-16T08:59:07,451 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38779 2024-11-16T08:59:07,452 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38779 2024-11-16T08:59:07,453 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38779 2024-11-16T08:59:07,454 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38779 2024-11-16T08:59:07,471 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;16bf49104b8f:35741 2024-11-16T08:59:07,472 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/16bf49104b8f,35741,1731747547220 2024-11-16T08:59:07,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:07,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:07,482 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:07,483 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/16bf49104b8f,35741,1731747547220 2024-11-16T08:59:07,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T08:59:07,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-16T08:59:07,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,494 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,495 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T08:59:07,495 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/16bf49104b8f,35741,1731747547220 from backup master directory 2024-11-16T08:59:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/16bf49104b8f,35741,1731747547220 2024-11-16T08:59:07,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-16T08:59:07,502 WARN [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T08:59:07,503 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=16bf49104b8f,35741,1731747547220 2024-11-16T08:59:07,504 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/hbase.id] with ID: 3274f7c1-2ed4-4818-95b8-c9ed2ca94371 2024-11-16T08:59:07,504 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.tmp/hbase.id 2024-11-16T08:59:07,507 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/hbase.id] 2024-11-16T08:59:07,536 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:07,536 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-16T08:59:07,539 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-16T08:59:07,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,553 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T08:59:07,555 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-16T08:59:07,555 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:07,580 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store 2024-11-16T08:59:07,589 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:07,589 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T08:59:07,589 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:07,589 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:07,589 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T08:59:07,589 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:07,590 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:07,590 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731747547589Disabling compacts and flushes for region at 1731747547589Disabling writes for close at 1731747547589Writing region close event to WAL at 1731747547590 (+1 ms)Closed at 1731747547590 2024-11-16T08:59:07,590 WARN [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/.initializing 2024-11-16T08:59:07,590 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/WALs/16bf49104b8f,35741,1731747547220 2024-11-16T08:59:07,598 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T08:59:07,602 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=16bf49104b8f%2C35741%2C1731747547220, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/WALs/16bf49104b8f,35741,1731747547220, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/oldWALs, maxLogs=10 2024-11-16T08:59:07,619 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/WALs/16bf49104b8f,35741,1731747547220/16bf49104b8f%2C35741%2C1731747547220.1731747547602 2024-11-16T08:59:07,620 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:07,620 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:07,620 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:07,620 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,620 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,624 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-16T08:59:07,624 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:07,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,628 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-16T08:59:07,628 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:07,629 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-16T08:59:07,632 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:07,632 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-16T08:59:07,636 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,636 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:07,636 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,637 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,637 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,637 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,637 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,637 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T08:59:07,638 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-16T08:59:07,643 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T08:59:07,643 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68130969, jitterRate=0.015230551362037659}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T08:59:07,644 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731747547620Initializing all the Stores at 1731747547620Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747547620Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747547620Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747547620Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747547620Cleaning up temporary data from old regions at 1731747547637 (+17 ms)Region opened successfully at 1731747547644 (+7 ms) 2024-11-16T08:59:07,645 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-16T08:59:07,648 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4399e619, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16bf49104b8f/172.17.0.2:0 2024-11-16T08:59:07,649 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-16T08:59:07,649 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-16T08:59:07,649 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-16T08:59:07,650 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-16T08:59:07,650 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-16T08:59:07,650 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-16T08:59:07,650 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-16T08:59:07,652 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-16T08:59:07,653 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-16T08:59:07,674 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-16T08:59:07,674 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-16T08:59:07,675 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-16T08:59:07,686 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-16T08:59:07,686 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-16T08:59:07,687 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-16T08:59:07,699 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-16T08:59:07,700 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-16T08:59:07,710 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-16T08:59:07,714 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-16T08:59:07,723 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-16T08:59:07,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:07,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:07,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:07,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,736 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,737 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=16bf49104b8f,35741,1731747547220, sessionid=0x10144281b0a0000, setting cluster-up flag (Was=false) 2024-11-16T08:59:07,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,786 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-16T08:59:07,788 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16bf49104b8f,35741,1731747547220 2024-11-16T08:59:07,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:07,836 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-16T08:59:07,837 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=16bf49104b8f,35741,1731747547220 2024-11-16T08:59:07,838 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-16T08:59:07,839 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:07,840 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-16T08:59:07,840 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-16T08:59:07,840 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 16bf49104b8f,35741,1731747547220 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/16bf49104b8f:0, corePoolSize=5, maxPoolSize=5 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/16bf49104b8f:0, corePoolSize=10, maxPoolSize=10 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/16bf49104b8f:0, corePoolSize=2, maxPoolSize=2 2024-11-16T08:59:07,842 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,843 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731747577843 2024-11-16T08:59:07,843 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-16T08:59:07,844 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-16T08:59:07,844 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-16T08:59:07,844 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-16T08:59:07,844 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-16T08:59:07,844 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-16T08:59:07,844 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:07,844 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-16T08:59:07,845 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,845 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-16T08:59:07,846 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,847 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-16T08:59:07,847 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-16T08:59:07,847 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-16T08:59:07,848 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-16T08:59:07,848 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-16T08:59:07,848 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.large.0-1731747547848,5,FailOnTimeoutGroup] 2024-11-16T08:59:07,849 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.small.0-1731747547849,5,FailOnTimeoutGroup] 2024-11-16T08:59:07,849 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,849 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-16T08:59:07,849 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,849 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,857 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(746): ClusterId : 3274f7c1-2ed4-4818-95b8-c9ed2ca94371 2024-11-16T08:59:07,857 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(746): ClusterId : 3274f7c1-2ed4-4818-95b8-c9ed2ca94371 2024-11-16T08:59:07,857 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T08:59:07,857 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T08:59:07,860 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-16T08:59:07,860 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:07,865 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:07,866 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T08:59:07,869 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T08:59:07,869 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:07,870 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T08:59:07,872 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T08:59:07,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:07,873 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T08:59:07,874 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T08:59:07,874 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T08:59:07,875 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T08:59:07,875 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T08:59:07,877 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T08:59:07,877 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,877 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:07,878 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T08:59:07,882 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T08:59:07,882 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:07,883 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:07,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T08:59:07,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:07,883 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:07,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T08:59:07,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T08:59:07,884 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T08:59:07,884 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T08:59:07,889 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T08:59:07,890 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71833468, jitterRate=0.07040208578109741}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T08:59:07,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731747547866Initializing all the Stores at 1731747547866Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747547866Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747547866Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747547866Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747547866Cleaning up temporary data from old regions at 1731747547884 (+18 ms)Region opened successfully at 1731747547890 (+6 ms) 2024-11-16T08:59:07,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T08:59:07,891 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T08:59:07,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T08:59:07,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T08:59:07,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T08:59:07,891 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T08:59:07,891 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731747547891Disabling compacts and flushes for region at 1731747547891Disabling writes for close at 1731747547891Writing region close event to WAL at 1731747547891Closed at 1731747547891 2024-11-16T08:59:07,893 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:07,893 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-16T08:59:07,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-16T08:59:07,894 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T08:59:07,895 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T08:59:07,895 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-16T08:59:07,895 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T08:59:07,895 DEBUG [RS:1;16bf49104b8f:38779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d5247a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16bf49104b8f/172.17.0.2:0 2024-11-16T08:59:07,895 DEBUG [RS:0;16bf49104b8f:42025 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c31a3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16bf49104b8f/172.17.0.2:0 2024-11-16T08:59:07,908 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;16bf49104b8f:42025 2024-11-16T08:59:07,908 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;16bf49104b8f:38779 2024-11-16T08:59:07,908 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T08:59:07,908 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T08:59:07,908 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T08:59:07,908 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T08:59:07,908 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T08:59:07,908 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T08:59:07,909 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(2659): reportForDuty to master=16bf49104b8f,35741,1731747547220 with port=42025, startcode=1731747547392 2024-11-16T08:59:07,909 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(2659): reportForDuty to master=16bf49104b8f,35741,1731747547220 with port=38779, startcode=1731747547431 2024-11-16T08:59:07,909 DEBUG [RS:0;16bf49104b8f:42025 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T08:59:07,909 DEBUG [RS:1;16bf49104b8f:38779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T08:59:07,912 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53719, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T08:59:07,912 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54749, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T08:59:07,913 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35741 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16bf49104b8f,38779,1731747547431 2024-11-16T08:59:07,913 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35741 {}] master.ServerManager(517): Registering regionserver=16bf49104b8f,38779,1731747547431 2024-11-16T08:59:07,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35741 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16bf49104b8f,42025,1731747547392 2024-11-16T08:59:07,915 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35741 {}] master.ServerManager(517): Registering regionserver=16bf49104b8f,42025,1731747547392 2024-11-16T08:59:07,915 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:07,915 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T08:59:07,915 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T08:59:07,916 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:07,916 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T08:59:07,916 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T08:59:07,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T08:59:07,956 DEBUG [RS:1;16bf49104b8f:38779 {}] zookeeper.ZKUtil(111): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16bf49104b8f,38779,1731747547431 2024-11-16T08:59:07,956 WARN [RS:1;16bf49104b8f:38779 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T08:59:07,956 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16bf49104b8f,38779,1731747547431] 2024-11-16T08:59:07,956 INFO [RS:1;16bf49104b8f:38779 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:07,956 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16bf49104b8f,42025,1731747547392] 2024-11-16T08:59:07,957 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,38779,1731747547431 2024-11-16T08:59:07,957 DEBUG [RS:0;16bf49104b8f:42025 {}] zookeeper.ZKUtil(111): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16bf49104b8f,42025,1731747547392 2024-11-16T08:59:07,957 WARN [RS:0;16bf49104b8f:42025 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T08:59:07,957 INFO [RS:0;16bf49104b8f:42025 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:07,957 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42025,1731747547392 2024-11-16T08:59:07,964 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T08:59:07,964 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T08:59:07,967 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T08:59:07,967 INFO [RS:1;16bf49104b8f:38779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T08:59:07,968 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,968 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T08:59:07,969 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T08:59:07,969 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,969 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,969 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,969 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,969 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16bf49104b8f:0, corePoolSize=2, maxPoolSize=2 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,970 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:07,971 DEBUG [RS:1;16bf49104b8f:38779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:07,971 INFO [RS:0;16bf49104b8f:42025 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T08:59:07,971 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,971 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T08:59:07,972 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T08:59:07,972 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,972 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,972 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16bf49104b8f:0, corePoolSize=2, maxPoolSize=2 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,973 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,973 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,38779,1731747547431-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T08:59:07,973 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:07,974 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:07,974 DEBUG [RS:0;16bf49104b8f:42025 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:07,974 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,974 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,974 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,974 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,975 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,975 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,42025,1731747547392-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T08:59:07,989 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T08:59:07,989 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T08:59:07,989 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,42025,1731747547392-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,989 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,38779,1731747547431-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,989 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,989 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.Replication(171): 16bf49104b8f,42025,1731747547392 started 2024-11-16T08:59:07,989 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:07,989 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.Replication(171): 16bf49104b8f,38779,1731747547431 started 2024-11-16T08:59:08,002 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,002 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,003 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1482): Serving as 16bf49104b8f,38779,1731747547431, RpcServer on 16bf49104b8f/172.17.0.2:38779, sessionid=0x10144281b0a0002 2024-11-16T08:59:08,003 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1482): Serving as 16bf49104b8f,42025,1731747547392, RpcServer on 16bf49104b8f/172.17.0.2:42025, sessionid=0x10144281b0a0001 2024-11-16T08:59:08,003 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T08:59:08,003 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T08:59:08,003 DEBUG [RS:1;16bf49104b8f:38779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16bf49104b8f,38779,1731747547431 2024-11-16T08:59:08,003 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,38779,1731747547431' 2024-11-16T08:59:08,003 DEBUG [RS:0;16bf49104b8f:42025 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16bf49104b8f,42025,1731747547392 2024-11-16T08:59:08,003 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T08:59:08,003 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,42025,1731747547392' 2024-11-16T08:59:08,003 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T08:59:08,004 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T08:59:08,004 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T08:59:08,004 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T08:59:08,004 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T08:59:08,004 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T08:59:08,004 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T08:59:08,004 DEBUG [RS:1;16bf49104b8f:38779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16bf49104b8f,38779,1731747547431 2024-11-16T08:59:08,004 DEBUG [RS:0;16bf49104b8f:42025 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16bf49104b8f,42025,1731747547392 2024-11-16T08:59:08,004 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,38779,1731747547431' 2024-11-16T08:59:08,004 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,42025,1731747547392' 2024-11-16T08:59:08,004 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T08:59:08,004 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T08:59:08,005 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T08:59:08,005 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T08:59:08,005 DEBUG [RS:1;16bf49104b8f:38779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T08:59:08,005 INFO [RS:1;16bf49104b8f:38779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T08:59:08,005 DEBUG [RS:0;16bf49104b8f:42025 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T08:59:08,005 INFO [RS:1;16bf49104b8f:38779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T08:59:08,005 INFO [RS:0;16bf49104b8f:42025 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T08:59:08,005 INFO [RS:0;16bf49104b8f:42025 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T08:59:08,045 WARN [16bf49104b8f:35741 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-16T08:59:08,107 INFO [RS:0;16bf49104b8f:42025 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T08:59:08,108 INFO [RS:1;16bf49104b8f:38779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T08:59:08,110 INFO [RS:0;16bf49104b8f:42025 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16bf49104b8f%2C42025%2C1731747547392, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42025,1731747547392, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs, maxLogs=55 2024-11-16T08:59:08,110 INFO [RS:1;16bf49104b8f:38779 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16bf49104b8f%2C38779%2C1731747547431, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,38779,1731747547431, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs, maxLogs=55 2024-11-16T08:59:08,132 INFO [RS:0;16bf49104b8f:42025 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42025,1731747547392/16bf49104b8f%2C42025%2C1731747547392.1731747548116 2024-11-16T08:59:08,132 INFO [RS:1;16bf49104b8f:38779 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,38779,1731747547431/16bf49104b8f%2C38779%2C1731747547431.1731747548116 2024-11-16T08:59:08,132 DEBUG [RS:0;16bf49104b8f:42025 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:08,132 DEBUG [RS:1;16bf49104b8f:38779 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:08,296 DEBUG [16bf49104b8f:35741 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-16T08:59:08,296 DEBUG [16bf49104b8f:35741 {}] balancer.BalancerClusterState(204): Hosts are {16bf49104b8f=0} racks are {/default-rack=0} 2024-11-16T08:59:08,298 DEBUG [16bf49104b8f:35741 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T08:59:08,298 DEBUG [16bf49104b8f:35741 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T08:59:08,298 DEBUG [16bf49104b8f:35741 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T08:59:08,298 DEBUG [16bf49104b8f:35741 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T08:59:08,298 INFO [16bf49104b8f:35741 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T08:59:08,298 INFO [16bf49104b8f:35741 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T08:59:08,298 DEBUG [16bf49104b8f:35741 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T08:59:08,299 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=16bf49104b8f,38779,1731747547431 2024-11-16T08:59:08,301 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16bf49104b8f,38779,1731747547431, state=OPENING 2024-11-16T08:59:08,336 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-16T08:59:08,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:08,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:08,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:08,345 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-16T08:59:08,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=16bf49104b8f,38779,1731747547431}] 2024-11-16T08:59:08,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:08,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:08,346 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:08,500 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T08:59:08,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51759, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T08:59:08,508 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-16T08:59:08,508 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:08,509 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-16T08:59:08,510 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16bf49104b8f%2C38779%2C1731747547431.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,38779,1731747547431, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs, maxLogs=55 2024-11-16T08:59:08,528 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,38779,1731747547431/16bf49104b8f%2C38779%2C1731747547431.meta.1731747548510.meta 2024-11-16T08:59:08,529 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:08,529 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:08,529 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-16T08:59:08,529 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-16T08:59:08,529 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-16T08:59:08,529 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-16T08:59:08,530 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:08,530 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-16T08:59:08,530 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-16T08:59:08,531 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-16T08:59:08,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-16T08:59:08,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:08,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:08,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-16T08:59:08,532 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-16T08:59:08,533 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:08,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:08,533 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-16T08:59:08,534 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-16T08:59:08,534 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:08,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:08,534 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-16T08:59:08,535 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-16T08:59:08,535 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:08,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-16T08:59:08,535 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-16T08:59:08,536 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:08,536 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740 2024-11-16T08:59:08,536 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-16T08:59:08,536 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-16T08:59:08,537 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-16T08:59:08,537 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-16T08:59:08,538 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73990235, jitterRate=0.10254041850566864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-16T08:59:08,538 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-16T08:59:08,538 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731747548530Writing region info on filesystem at 1731747548530Initializing all the Stores at 1731747548530Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747548530Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747548530Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747548530Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731747548530Cleaning up temporary data from old regions at 1731747548536 (+6 ms)Running coprocessor post-open hooks at 1731747548538 (+2 ms)Region opened successfully at 1731747548538 2024-11-16T08:59:08,539 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731747548499 2024-11-16T08:59:08,542 DEBUG [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-16T08:59:08,542 INFO [RS_OPEN_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-16T08:59:08,542 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=16bf49104b8f,38779,1731747547431 2024-11-16T08:59:08,543 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 16bf49104b8f,38779,1731747547431, state=OPEN 2024-11-16T08:59:08,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T08:59:08,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T08:59:08,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-16T08:59:08,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:08,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:08,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-16T08:59:08,574 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=16bf49104b8f,38779,1731747547431 2024-11-16T08:59:08,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-16T08:59:08,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=16bf49104b8f,38779,1731747547431 in 229 msec 2024-11-16T08:59:08,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-16T08:59:08,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 685 msec 2024-11-16T08:59:08,582 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-16T08:59:08,582 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-16T08:59:08,585 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T08:59:08,585 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16bf49104b8f,38779,1731747547431, seqNum=-1] 2024-11-16T08:59:08,585 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T08:59:08,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38035, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T08:59:08,591 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 752 msec 2024-11-16T08:59:08,591 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731747548591, completionTime=-1 2024-11-16T08:59:08,591 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-16T08:59:08,592 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-16T08:59:08,594 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-16T08:59:08,594 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731747608594 2024-11-16T08:59:08,594 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731747668594 2024-11-16T08:59:08,594 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-16T08:59:08,594 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-16T08:59:08,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,35741,1731747547220-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,35741,1731747547220-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,35741,1731747547220-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-16bf49104b8f:35741, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,595 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,598 DEBUG [master/16bf49104b8f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-16T08:59:08,600 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.097sec 2024-11-16T08:59:08,601 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-16T08:59:08,601 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-16T08:59:08,601 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-16T08:59:08,601 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-16T08:59:08,601 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-16T08:59:08,601 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,35741,1731747547220-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T08:59:08,601 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,35741,1731747547220-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-16T08:59:08,602 DEBUG [master/16bf49104b8f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-16T08:59:08,602 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-16T08:59:08,602 INFO [master/16bf49104b8f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,35741,1731747547220-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:08,658 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2129cce9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T08:59:08,658 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 16bf49104b8f,35741,-1 for getting cluster id 2024-11-16T08:59:08,658 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-16T08:59:08,660 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3274f7c1-2ed4-4818-95b8-c9ed2ca94371' 2024-11-16T08:59:08,661 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-16T08:59:08,661 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3274f7c1-2ed4-4818-95b8-c9ed2ca94371" 2024-11-16T08:59:08,662 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f25ecfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T08:59:08,662 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [16bf49104b8f,35741,-1] 2024-11-16T08:59:08,662 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-16T08:59:08,662 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:08,664 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35738, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-16T08:59:08,665 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e62305b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-16T08:59:08,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-16T08:59:08,667 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=16bf49104b8f,38779,1731747547431, seqNum=-1] 2024-11-16T08:59:08,668 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T08:59:08,670 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41516, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T08:59:08,673 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=16bf49104b8f,35741,1731747547220 2024-11-16T08:59:08,673 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-16T08:59:08,675 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 16bf49104b8f,35741,1731747547220 2024-11-16T08:59:08,675 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@44bf9d7d 2024-11-16T08:59:08,675 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-16T08:59:08,677 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-16T08:59:08,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35741 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731747548673', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-16T08:59:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35741 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 2024-11-16T08:59:08,680 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-16T08:59:08,680 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:08,681 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35741 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731747548673" procId is: 4 2024-11-16T08:59:08,681 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-16T08:59:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:08,690 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => eeacf949c2d9f31f7badb81aed6ad472, NAME => 'testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731747548673', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:08,696 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:08,696 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(1722): Closing eeacf949c2d9f31f7badb81aed6ad472, disabling compactions & flushes 2024-11-16T08:59:08,696 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:08,696 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:08,696 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. after waiting 0 ms 2024-11-16T08:59:08,696 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:08,696 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:08,697 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731747548673-pool-0 {}] regionserver.HRegion(1676): Region close journal for eeacf949c2d9f31f7badb81aed6ad472: Waiting for close lock at 1731747548696Disabling compacts and flushes for region at 1731747548696Disabling writes for close at 1731747548696Writing region close event to WAL at 1731747548696Closed at 1731747548696 2024-11-16T08:59:08,698 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 execute state=CREATE_TABLE_ADD_TO_META 2024-11-16T08:59:08,698 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731747548698"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731747548698"}]},"ts":"1731747548698"} 2024-11-16T08:59:08,700 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-16T08:59:08,701 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-16T08:59:08,702 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731747548673","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731747548701"}]},"ts":"1731747548701"} 2024-11-16T08:59:08,704 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731747548673, state=ENABLING in hbase:meta 2024-11-16T08:59:08,704 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {16bf49104b8f=0} racks are {/default-rack=0} 2024-11-16T08:59:08,704 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-16T08:59:08,704 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-16T08:59:08,704 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-16T08:59:08,704 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-16T08:59:08,705 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-16T08:59:08,705 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-16T08:59:08,705 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-16T08:59:08,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN}] 2024-11-16T08:59:08,706 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN 2024-11-16T08:59:08,707 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN; state=OFFLINE, location=16bf49104b8f,42025,1731747547392; forceNewPlan=false, retain=false 2024-11-16T08:59:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:08,857 INFO [16bf49104b8f:35741 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-16T08:59:08,858 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eeacf949c2d9f31f7badb81aed6ad472, regionState=OPENING, regionLocation=16bf49104b8f,42025,1731747547392 2024-11-16T08:59:08,862 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN because future has completed 2024-11-16T08:59:08,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeacf949c2d9f31f7badb81aed6ad472, server=16bf49104b8f,42025,1731747547392}] 2024-11-16T08:59:09,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:09,017 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-16T08:59:09,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54847, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-16T08:59:09,023 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:09,024 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => eeacf949c2d9f31f7badb81aed6ad472, NAME => 'testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:09,024 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731747548673 eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,025 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:09,025 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,025 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,027 INFO [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,030 INFO [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeacf949c2d9f31f7badb81aed6ad472 columnFamilyName family 2024-11-16T08:59:09,031 DEBUG [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:09,031 INFO [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] regionserver.HStore(327): Store=eeacf949c2d9f31f7badb81aed6ad472/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:09,031 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,032 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,032 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,032 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,032 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,033 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,039 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-16T08:59:09,039 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened eeacf949c2d9f31f7badb81aed6ad472; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73979422, jitterRate=0.10237929224967957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T08:59:09,040 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:09,041 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for eeacf949c2d9f31f7badb81aed6ad472: Running coprocessor pre-open hook at 1731747549025Writing region info on filesystem at 1731747549025Initializing all the Stores at 1731747549026 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747549026Cleaning up temporary data from old regions at 1731747549032 (+6 ms)Running coprocessor post-open hooks at 1731747549040 (+8 ms)Region opened successfully at 1731747549041 (+1 ms) 2024-11-16T08:59:09,042 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472., pid=6, masterSystemTime=1731747549016 2024-11-16T08:59:09,045 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:09,046 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:09,046 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=eeacf949c2d9f31f7badb81aed6ad472, regionState=OPEN, openSeqNum=2, regionLocation=16bf49104b8f,42025,1731747547392 2024-11-16T08:59:09,049 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeacf949c2d9f31f7badb81aed6ad472, server=16bf49104b8f,42025,1731747547392 because future has completed 2024-11-16T08:59:09,053 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-16T08:59:09,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure eeacf949c2d9f31f7badb81aed6ad472, server=16bf49104b8f,42025,1731747547392 in 189 msec 2024-11-16T08:59:09,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-16T08:59:09,056 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN in 349 msec 2024-11-16T08:59:09,057 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-16T08:59:09,057 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731747548673","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731747549057"}]},"ts":"1731747549057"} 2024-11-16T08:59:09,060 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731747548673, state=ENABLED in hbase:meta 2024-11-16T08:59:09,061 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 execute state=CREATE_TABLE_POST_OPERATION 2024-11-16T08:59:09,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731747548673 in 383 msec 2024-11-16T08:59:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35741 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-16T08:59:09,316 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731747548673 completed 2024-11-16T08:59:09,316 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731747548673 get assigned. Timeout = 60000ms 2024-11-16T08:59:09,316 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T08:59:09,319 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731747548673 assigned to meta. Checking AM states. 2024-11-16T08:59:09,320 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-16T08:59:09,320 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731747548673 assigned. 2024-11-16T08:59:09,324 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731747548673', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472., hostname=16bf49104b8f,42025,1731747547392, seqNum=2] 2024-11-16T08:59:09,324 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-16T08:59:09,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-16T08:59:09,332 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing eeacf949c2d9f31f7badb81aed6ad472 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-16T08:59:09,343 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 97ba1725200e48c1af109dbdc416696e 2024-11-16T08:59:09,359 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/.tmp/family/97ba1725200e48c1af109dbdc416696e is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-16T08:59:09,360 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/.tmp/family/97ba1725200e48c1af109dbdc416696e 2024-11-16T08:59:09,362 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/.tmp/family/97ba1725200e48c1af109dbdc416696e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e 2024-11-16T08:59:09,363 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e, delay=958 ms 2024-11-16T08:59:09,365 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e, entries=2, sequenceid=6, filesize=4.9 K 2024-11-16T08:59:09,365 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for eeacf949c2d9f31f7badb81aed6ad472 in 33ms, sequenceid=6, compaction requested=false 2024-11-16T08:59:09,365 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731747548673' 2024-11-16T08:59:09,366 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for eeacf949c2d9f31f7badb81aed6ad472: 2024-11-16T08:59:09,371 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-16T08:59:10,323 DEBUG [hfile-prefetch-1731747546556 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 97ba1725200e48c1af109dbdc416696e_0. 2024-11-16T08:59:10,324 DEBUG [hfile-prefetch-1731747546556 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 97ba1725200e48c1af109dbdc416696e_117. 2024-11-16T08:59:10,325 DEBUG [hfile-prefetch-1731747546556 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e, with total blocks 2, and data blocks 1 2024-11-16T08:59:10,325 DEBUG [hfile-prefetch-1731747546556 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-16T08:59:10,325 DEBUG [hfile-prefetch-1731747546556 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-16T08:59:10,325 DEBUG [hfile-prefetch-1731747546556 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e. Will sleep 100ms and try the verification again. 2024-11-16T08:59:10,366 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;16bf49104b8f:42025,5,FailOnTimeoutGroup] 2024-11-16T08:59:10,367 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16bf49104b8f,42025,1731747547392' ***** 2024-11-16T08:59:10,367 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-16T08:59:10,367 INFO [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T08:59:10,377 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T08:59:10,377 INFO [regionserver/16bf49104b8f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:10,397 INFO [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence 2024-11-16T08:59:10,397 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T08:59:10,397 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T08:59:10,397 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T08:59:10,398 INFO [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T08:59:10,398 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T08:59:10,408 DEBUG [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1487): Thread RS:0;16bf49104b8f:42025 started persisting bucket cache to file 2024-11-16T08:59:10,408 DEBUG [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:10,408 DEBUG [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:10,412 DEBUG [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-16T08:59:10,412 DEBUG [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731747550408 2024-11-16T08:59:10,412 DEBUG [RS:0;16bf49104b8f:42025 {}] bucket.BucketCache(1509): Thread RS:0;16bf49104b8f:42025 finished persisting bucket cache to file, renaming 2024-11-16T08:59:10,413 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T08:59:10,413 INFO [RS:0;16bf49104b8f:42025 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T08:59:10,413 INFO [RS:0;16bf49104b8f:42025 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T08:59:10,413 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(3091): Received CLOSE for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:10,413 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(959): stopping server 16bf49104b8f,42025,1731747547392 2024-11-16T08:59:10,413 INFO [RS:0;16bf49104b8f:42025 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T08:59:10,414 INFO [RS:0;16bf49104b8f:42025 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;16bf49104b8f:42025. 2024-11-16T08:59:10,414 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T08:59:10,414 DEBUG [RS:0;16bf49104b8f:42025 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:10,414 DEBUG [RS:0;16bf49104b8f:42025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:10,414 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eeacf949c2d9f31f7badb81aed6ad472, disabling compactions & flushes 2024-11-16T08:59:10,414 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-16T08:59:10,414 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:10,414 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1325): Online Regions={eeacf949c2d9f31f7badb81aed6ad472=testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.} 2024-11-16T08:59:10,414 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:10,414 DEBUG [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1351): Waiting on eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:10,414 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. after waiting 0 ms 2024-11-16T08:59:10,415 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:10,416 DEBUG [StoreFileCloser-eeacf949c2d9f31f7badb81aed6ad472-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e 2024-11-16T08:59:10,418 WARN [hfile-prefetch-1731747546556 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-16T08:59:10,419 DEBUG [hfile-prefetch-1731747546556 {}] hfile.PrefetchExecutor(121): Prefetch completed for 97ba1725200e48c1af109dbdc416696e 2024-11-16T08:59:10,421 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-16T08:59:10,422 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:10,422 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eeacf949c2d9f31f7badb81aed6ad472: Waiting for close lock at 1731747550413Running coprocessor pre-close hooks at 1731747550414 (+1 ms)Disabling compacts and flushes for region at 1731747550414Disabling writes for close at 1731747550414Writing region close event to WAL at 1731747550416 (+2 ms)Running coprocessor post-close hooks at 1731747550422 (+6 ms)Closed at 1731747550422 2024-11-16T08:59:10,423 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:10,449 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-16T08:59:10,449 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-16T08:59:10,449 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:10,449 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:10,452 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:10,452 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747550449 2024-11-16T08:59:10,452 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-16T08:59:10,615 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(976): stopping server 16bf49104b8f,42025,1731747547392; all regions closed. 2024-11-16T08:59:10,621 DEBUG [RS:0;16bf49104b8f:42025 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs 2024-11-16T08:59:10,621 INFO [RS:0;16bf49104b8f:42025 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16bf49104b8f%2C42025%2C1731747547392:(num 1731747548116) 2024-11-16T08:59:10,621 DEBUG [RS:0;16bf49104b8f:42025 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:10,621 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:10,622 INFO [RS:0;16bf49104b8f:42025 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T08:59:10,622 INFO [RS:0;16bf49104b8f:42025 {}] hbase.ChoreService(370): Chore service for: regionserver/16bf49104b8f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-16T08:59:10,623 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T08:59:10,623 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T08:59:10,623 INFO [regionserver/16bf49104b8f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T08:59:10,623 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T08:59:10,623 INFO [RS:0;16bf49104b8f:42025 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T08:59:10,623 INFO [RS:0;16bf49104b8f:42025 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42025 2024-11-16T08:59:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T08:59:10,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16bf49104b8f,42025,1731747547392 2024-11-16T08:59:10,652 INFO [RS:0;16bf49104b8f:42025 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T08:59:10,660 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16bf49104b8f,42025,1731747547392] 2024-11-16T08:59:10,669 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16bf49104b8f,42025,1731747547392 already deleted, retry=false 2024-11-16T08:59:10,669 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 16bf49104b8f,42025,1731747547392 on 16bf49104b8f,35741,1731747547220 2024-11-16T08:59:10,675 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 16bf49104b8f,42025,1731747547392, splitWal=true, meta=false 2024-11-16T08:59:10,677 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 16bf49104b8f,42025,1731747547392 (carryingMeta=false) 16bf49104b8f,42025,1731747547392/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@16524b08[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-16T08:59:10,677 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 16bf49104b8f,42025,1731747547392, splitWal=true, meta=false 2024-11-16T08:59:10,679 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): 16bf49104b8f,42025,1731747547392 had 1 regions 2024-11-16T08:59:10,680 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 16bf49104b8f,42025,1731747547392, splitWal=true, meta=false, isMeta: false 2024-11-16T08:59:10,681 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42025,1731747547392-splitting 2024-11-16T08:59:10,681 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42025,1731747547392-splitting dir is empty, no logs to split. 2024-11-16T08:59:10,681 INFO [PEWorker-2 {}] master.SplitWALManager(105): 16bf49104b8f,42025,1731747547392 WAL count=0, meta=false 2024-11-16T08:59:10,683 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42025,1731747547392-splitting dir is empty, no logs to split. 2024-11-16T08:59:10,684 INFO [PEWorker-2 {}] master.SplitWALManager(105): 16bf49104b8f,42025,1731747547392 WAL count=0, meta=false 2024-11-16T08:59:10,684 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 16bf49104b8f,42025,1731747547392 WAL splitting is done? wals=0, meta=false 2024-11-16T08:59:10,684 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 16bf49104b8f,42025,1731747547392 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42025,1731747547392-splitting does not exist 2024-11-16T08:59:10,686 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN}] 2024-11-16T08:59:10,687 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN 2024-11-16T08:59:10,688 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-16T08:59:10,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:10,761 INFO [RS:0;16bf49104b8f:42025 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T08:59:10,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42025-0x10144281b0a0001, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:10,761 INFO [RS:0;16bf49104b8f:42025 {}] regionserver.HRegionServer(1031): Exiting; stopping=16bf49104b8f,42025,1731747547392; zookeeper connection closed. 2024-11-16T08:59:10,761 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3ae782aa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3ae782aa 2024-11-16T08:59:10,840 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eeacf949c2d9f31f7badb81aed6ad472, regionState=OPENING, regionLocation=16bf49104b8f,38779,1731747547431 2024-11-16T08:59:10,844 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN because future has completed 2024-11-16T08:59:10,845 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeacf949c2d9f31f7badb81aed6ad472, server=16bf49104b8f,38779,1731747547431}] 2024-11-16T08:59:10,882 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/16bf49104b8f:0 server-side Connection retries=45 2024-11-16T08:59:10,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:10,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:10,883 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-16T08:59:10,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-16T08:59:10,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-16T08:59:10,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-16T08:59:10,883 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-16T08:59:10,884 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42921 2024-11-16T08:59:10,885 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42921 connecting to ZooKeeper ensemble=127.0.0.1:61244 2024-11-16T08:59:10,885 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:10,886 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:10,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429210x0, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-16T08:59:10,903 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-16T08:59:10,903 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:429210x0, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-16T08:59:10,903 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42921-0x10144281b0a0003 connected 2024-11-16T08:59:10,903 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-16T08:59:10,905 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.cache 2024-11-16T08:59:10,905 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-16T08:59:10,905 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-16T08:59:10,905 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-16T08:59:10,905 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-16T08:59:10,906 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-16T08:59:10,906 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-16T08:59:10,906 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-16T08:59:10,907 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-16T08:59:10,907 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-16T08:59:10,909 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-16T08:59:10,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42921 2024-11-16T08:59:10,909 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42921 2024-11-16T08:59:10,909 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-16T08:59:10,910 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-16T08:59:10,910 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-16T08:59:10,910 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence is complete. 2024-11-16T08:59:10,910 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42921 2024-11-16T08:59:10,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42921 2024-11-16T08:59:10,911 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42921 2024-11-16T08:59:10,912 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(746): ClusterId : 3274f7c1-2ed4-4818-95b8-c9ed2ca94371 2024-11-16T08:59:10,912 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-16T08:59:10,924 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-16T08:59:10,924 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-16T08:59:10,936 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-16T08:59:10,937 DEBUG [RS:2;16bf49104b8f:42921 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5725146b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=16bf49104b8f/172.17.0.2:0 2024-11-16T08:59:10,949 DEBUG [RS:2;16bf49104b8f:42921 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;16bf49104b8f:42921 2024-11-16T08:59:10,949 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-16T08:59:10,949 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-16T08:59:10,949 DEBUG [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-16T08:59:10,950 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(2659): reportForDuty to master=16bf49104b8f,35741,1731747547220 with port=42921, startcode=1731747550882 2024-11-16T08:59:10,950 DEBUG [RS:2;16bf49104b8f:42921 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-16T08:59:10,952 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48263, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-16T08:59:10,952 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35741 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 16bf49104b8f,42921,1731747550882 2024-11-16T08:59:10,952 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35741 {}] master.ServerManager(517): Registering regionserver=16bf49104b8f,42921,1731747550882 2024-11-16T08:59:10,954 DEBUG [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8 2024-11-16T08:59:10,954 DEBUG [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-16T08:59:10,954 DEBUG [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-16T08:59:10,999 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,000 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => eeacf949c2d9f31f7badb81aed6ad472, NAME => 'testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.', STARTKEY => '', ENDKEY => ''} 2024-11-16T08:59:11,001 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731747548673 eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,001 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-16T08:59:11,001 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,001 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,002 INFO [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,003 INFO [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eeacf949c2d9f31f7badb81aed6ad472 columnFamilyName family 2024-11-16T08:59:11,004 DEBUG [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-16T08:59:11,006 DEBUG [StoreFileOpener-eeacf949c2d9f31f7badb81aed6ad472-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e, delay=900 ms 2024-11-16T08:59:11,008 DEBUG [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e 2024-11-16T08:59:11,008 INFO [StoreOpener-eeacf949c2d9f31f7badb81aed6ad472-1 {}] regionserver.HStore(327): Store=eeacf949c2d9f31f7badb81aed6ad472/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-16T08:59:11,008 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,009 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,010 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,010 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,010 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,011 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,012 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened eeacf949c2d9f31f7badb81aed6ad472; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73686103, jitterRate=0.09800849854946136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-16T08:59:11,012 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,012 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for eeacf949c2d9f31f7badb81aed6ad472: Running coprocessor pre-open hook at 1731747551001Writing region info on filesystem at 1731747551001Initializing all the Stores at 1731747551002 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731747551002Cleaning up temporary data from old regions at 1731747551010 (+8 ms)Running coprocessor post-open hooks at 1731747551012 (+2 ms)Region opened successfully at 1731747551012 2024-11-16T08:59:11,013 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472., pid=9, masterSystemTime=1731747550997 2024-11-16T08:59:11,015 DEBUG [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,015 INFO [RS_OPEN_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,016 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=eeacf949c2d9f31f7badb81aed6ad472, regionState=OPEN, openSeqNum=10, regionLocation=16bf49104b8f,38779,1731747547431 2024-11-16T08:59:11,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure eeacf949c2d9f31f7badb81aed6ad472, server=16bf49104b8f,38779,1731747547431 because future has completed 2024-11-16T08:59:11,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-16T08:59:11,020 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure eeacf949c2d9f31f7badb81aed6ad472, server=16bf49104b8f,38779,1731747547431 in 174 msec 2024-11-16T08:59:11,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-16T08:59:11,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731747548673, region=eeacf949c2d9f31f7badb81aed6ad472, ASSIGN in 334 msec 2024-11-16T08:59:11,022 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 16bf49104b8f,42025,1731747547392 after splitting done 2024-11-16T08:59:11,023 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 16bf49104b8f,42025,1731747547392, splitWal=true, meta=false in 352 msec 2024-11-16T08:59:11,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T08:59:11,146 DEBUG [RS:2;16bf49104b8f:42921 {}] zookeeper.ZKUtil(111): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/16bf49104b8f,42921,1731747550882 2024-11-16T08:59:11,146 WARN [RS:2;16bf49104b8f:42921 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-16T08:59:11,147 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [16bf49104b8f,42921,1731747550882] 2024-11-16T08:59:11,147 INFO [RS:2;16bf49104b8f:42921 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-16T08:59:11,147 DEBUG [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42921,1731747550882 2024-11-16T08:59:11,157 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-16T08:59:11,160 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-16T08:59:11,160 INFO [RS:2;16bf49104b8f:42921 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-16T08:59:11,161 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,161 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-16T08:59:11,161 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-16T08:59:11,161 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/16bf49104b8f:0, corePoolSize=2, maxPoolSize=2 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,162 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,163 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,163 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,163 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,163 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/16bf49104b8f:0, corePoolSize=1, maxPoolSize=1 2024-11-16T08:59:11,163 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:11,163 DEBUG [RS:2;16bf49104b8f:42921 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/16bf49104b8f:0, corePoolSize=3, maxPoolSize=3 2024-11-16T08:59:11,163 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,163 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,164 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,164 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,164 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,164 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,42921,1731747550882-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-16T08:59:11,178 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-16T08:59:11,178 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=16bf49104b8f,42921,1731747550882-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,179 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,179 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.Replication(171): 16bf49104b8f,42921,1731747550882 started 2024-11-16T08:59:11,191 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-16T08:59:11,192 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(1482): Serving as 16bf49104b8f,42921,1731747550882, RpcServer on 16bf49104b8f/172.17.0.2:42921, sessionid=0x10144281b0a0003 2024-11-16T08:59:11,192 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-16T08:59:11,192 DEBUG [RS:2;16bf49104b8f:42921 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 16bf49104b8f,42921,1731747550882 2024-11-16T08:59:11,192 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,42921,1731747550882' 2024-11-16T08:59:11,192 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-16T08:59:11,193 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-16T08:59:11,193 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-16T08:59:11,193 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-16T08:59:11,193 DEBUG [RS:2;16bf49104b8f:42921 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 16bf49104b8f,42921,1731747550882 2024-11-16T08:59:11,193 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '16bf49104b8f,42921,1731747550882' 2024-11-16T08:59:11,193 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-16T08:59:11,194 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-16T08:59:11,194 DEBUG [RS:2;16bf49104b8f:42921 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-16T08:59:11,194 INFO [RS:2;16bf49104b8f:42921 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-16T08:59:11,194 INFO [RS:2;16bf49104b8f:42921 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-16T08:59:11,296 INFO [RS:2;16bf49104b8f:42921 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-16T08:59:11,297 INFO [RS:2;16bf49104b8f:42921 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=16bf49104b8f%2C42921%2C1731747550882, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42921,1731747550882, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs, maxLogs=55 2024-11-16T08:59:11,318 INFO [RS:2;16bf49104b8f:42921 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/WALs/16bf49104b8f,42921,1731747550882/16bf49104b8f%2C42921%2C1731747550882.1731747551298 2024-11-16T08:59:11,319 DEBUG [RS:2;16bf49104b8f:42921 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-16T08:59:11,384 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-16T08:59:11,385 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-16T08:59:11,387 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731747548673 2024-11-16T08:59:11,387 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1731747548673 Metrics about Tables on a single HBase RegionServer 2024-11-16T08:59:11,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T08:59:11,388 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-16T08:59:11,388 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-16T08:59:11,388 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-16T08:59:11,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-16T08:59:11,692 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T08:59:11,692 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:11,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:11,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:11,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-16T08:59:11,693 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-16T08:59:11,693 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=493163895, stopped=false 2024-11-16T08:59:11,693 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=16bf49104b8f,35741,1731747547220 2024-11-16T08:59:11,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:11,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:11,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-16T08:59:11,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:11,757 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T08:59:11,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:11,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:11,757 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-16T08:59:11,758 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:11,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:11,759 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16bf49104b8f,38779,1731747547431' ***** 2024-11-16T08:59:11,759 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T08:59:11,759 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '16bf49104b8f,42921,1731747550882' ***** 2024-11-16T08:59:11,759 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:11,759 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-16T08:59:11,759 INFO [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T08:59:11,759 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:11,760 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-16T08:59:11,760 INFO [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence 2024-11-16T08:59:11,760 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T08:59:11,760 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T08:59:11,760 INFO [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1738): Disabling cache 2024-11-16T08:59:11,760 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T08:59:11,761 INFO [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/00618538-4190-b3db-f3cc-279b10655cca/bucket.persistence 2024-11-16T08:59:11,761 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-16T08:59:11,761 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-16T08:59:11,761 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-16T08:59:11,761 INFO [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T08:59:11,761 INFO [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-16T08:59:11,761 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T08:59:11,762 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-16T08:59:11,762 DEBUG [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1487): Thread RS:1;16bf49104b8f:38779 started persisting bucket cache to file 2024-11-16T08:59:11,762 DEBUG [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:11,763 DEBUG [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:11,764 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T08:59:11,765 INFO [regionserver/16bf49104b8f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:11,766 DEBUG [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:11,766 DEBUG [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747551762 2024-11-16T08:59:11,766 DEBUG [RS:1;16bf49104b8f:38779 {}] bucket.BucketCache(1509): Thread RS:1;16bf49104b8f:38779 finished persisting bucket cache to file, renaming 2024-11-16T08:59:11,766 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T08:59:11,766 INFO [RS:1;16bf49104b8f:38779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T08:59:11,766 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-16T08:59:11,766 INFO [RS:1;16bf49104b8f:38779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T08:59:11,766 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,767 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(959): stopping server 16bf49104b8f,38779,1731747547431 2024-11-16T08:59:11,767 INFO [RS:1;16bf49104b8f:38779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T08:59:11,767 INFO [RS:1;16bf49104b8f:38779 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;16bf49104b8f:38779. 2024-11-16T08:59:11,767 DEBUG [RS:1;16bf49104b8f:38779 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:11,767 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T08:59:11,767 DEBUG [RS:1;16bf49104b8f:38779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:11,767 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eeacf949c2d9f31f7badb81aed6ad472, disabling compactions & flushes 2024-11-16T08:59:11,767 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T08:59:11,767 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,767 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T08:59:11,767 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T08:59:11,767 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,767 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-16T08:59:11,767 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. after waiting 0 ms 2024-11-16T08:59:11,767 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,768 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-16T08:59:11,768 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1325): Online Regions={eeacf949c2d9f31f7badb81aed6ad472=testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472., 1588230740=hbase:meta,,1.1588230740} 2024-11-16T08:59:11,768 DEBUG [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, eeacf949c2d9f31f7badb81aed6ad472 2024-11-16T08:59:11,768 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-16T08:59:11,768 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-16T08:59:11,768 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-16T08:59:11,768 DEBUG [StoreFileCloser-eeacf949c2d9f31f7badb81aed6ad472-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/family/97ba1725200e48c1af109dbdc416696e 2024-11-16T08:59:11,768 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-16T08:59:11,768 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-16T08:59:11,768 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-16T08:59:11,768 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-16T08:59:11,771 DEBUG [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1487): Thread RS:2;16bf49104b8f:42921 started persisting bucket cache to file 2024-11-16T08:59:11,772 DEBUG [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-16T08:59:11,772 DEBUG [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-16T08:59:11,772 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/default/testBlockEvictionOnGracefulStop1731747548673/eeacf949c2d9f31f7badb81aed6ad472/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-16T08:59:11,772 INFO [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,772 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eeacf949c2d9f31f7badb81aed6ad472: Waiting for close lock at 1731747551767Running coprocessor pre-close hooks at 1731747551767Disabling compacts and flushes for region at 1731747551767Disabling writes for close at 1731747551767Writing region close event to WAL at 1731747551768 (+1 ms)Running coprocessor post-close hooks at 1731747551772 (+4 ms)Closed at 1731747551772 2024-11-16T08:59:11,773 DEBUG [RS_CLOSE_REGION-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472. 2024-11-16T08:59:11,774 DEBUG [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-16T08:59:11,774 DEBUG [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731747551772 2024-11-16T08:59:11,775 DEBUG [RS:2;16bf49104b8f:42921 {}] bucket.BucketCache(1509): Thread RS:2;16bf49104b8f:42921 finished persisting bucket cache to file, renaming 2024-11-16T08:59:11,775 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-16T08:59:11,775 INFO [RS:2;16bf49104b8f:42921 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-16T08:59:11,775 INFO [RS:2;16bf49104b8f:42921 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-16T08:59:11,775 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(959): stopping server 16bf49104b8f,42921,1731747550882 2024-11-16T08:59:11,775 INFO [RS:2;16bf49104b8f:42921 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T08:59:11,775 INFO [RS:2;16bf49104b8f:42921 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;16bf49104b8f:42921. 2024-11-16T08:59:11,775 DEBUG [RS:2;16bf49104b8f:42921 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-16T08:59:11,775 DEBUG [RS:2;16bf49104b8f:42921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:11,775 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(976): stopping server 16bf49104b8f,42921,1731747550882; all regions closed. 2024-11-16T08:59:11,777 DEBUG [RS:2;16bf49104b8f:42921 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs 2024-11-16T08:59:11,777 INFO [RS:2;16bf49104b8f:42921 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16bf49104b8f%2C42921%2C1731747550882:(num 1731747551298) 2024-11-16T08:59:11,777 DEBUG [RS:2;16bf49104b8f:42921 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:11,777 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:11,777 INFO [RS:2;16bf49104b8f:42921 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T08:59:11,777 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f989e87e75c14fa8bd62d443ac381f5a 2024-11-16T08:59:11,777 INFO [RS:2;16bf49104b8f:42921 {}] hbase.ChoreService(370): Chore service for: regionserver/16bf49104b8f:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T08:59:11,778 INFO [regionserver/16bf49104b8f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:11,778 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-16T08:59:11,778 INFO [regionserver/16bf49104b8f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T08:59:11,778 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-16T08:59:11,778 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-16T08:59:11,778 INFO [RS:2;16bf49104b8f:42921 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T08:59:11,778 INFO [RS:2;16bf49104b8f:42921 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42921 2024-11-16T08:59:11,791 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/info/f989e87e75c14fa8bd62d443ac381f5a is 209, key is testBlockEvictionOnGracefulStop1731747548673,,1731747548678.eeacf949c2d9f31f7badb81aed6ad472./info:regioninfo/1731747551016/Put/seqid=0 2024-11-16T08:59:11,791 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/info/f989e87e75c14fa8bd62d443ac381f5a 2024-11-16T08:59:11,798 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7273e93eb3a2408ab3aaacb095ec0a0e 2024-11-16T08:59:11,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T08:59:11,802 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16bf49104b8f,42921,1731747550882 2024-11-16T08:59:11,802 INFO [RS:2;16bf49104b8f:42921 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T08:59:11,803 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16bf49104b8f,42921,1731747550882] 2024-11-16T08:59:11,812 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/ns/7273e93eb3a2408ab3aaacb095ec0a0e is 43, key is default/ns:d/1731747548587/Put/seqid=0 2024-11-16T08:59:11,812 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/ns/7273e93eb3a2408ab3aaacb095ec0a0e 2024-11-16T08:59:11,818 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16bf49104b8f,42921,1731747550882 already deleted, retry=false 2024-11-16T08:59:11,818 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16bf49104b8f,42921,1731747550882 expired; onlineServers=1 2024-11-16T08:59:11,820 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 88b7aa6e22eb4a6ba5bd2f884de0de55 2024-11-16T08:59:11,832 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/table/88b7aa6e22eb4a6ba5bd2f884de0de55 is 80, key is testBlockEvictionOnGracefulStop1731747548673/table:state/1731747549057/Put/seqid=0 2024-11-16T08:59:11,833 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/table/88b7aa6e22eb4a6ba5bd2f884de0de55 2024-11-16T08:59:11,835 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/info/f989e87e75c14fa8bd62d443ac381f5a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/f989e87e75c14fa8bd62d443ac381f5a 2024-11-16T08:59:11,836 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/f989e87e75c14fa8bd62d443ac381f5a, delay=975 ms 2024-11-16T08:59:11,837 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/f989e87e75c14fa8bd62d443ac381f5a, entries=14, sequenceid=13, filesize=7.5 K 2024-11-16T08:59:11,838 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/ns/7273e93eb3a2408ab3aaacb095ec0a0e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/7273e93eb3a2408ab3aaacb095ec0a0e 2024-11-16T08:59:11,839 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/7273e93eb3a2408ab3aaacb095ec0a0e, delay=961 ms 2024-11-16T08:59:11,839 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/7273e93eb3a2408ab3aaacb095ec0a0e, entries=2, sequenceid=13, filesize=5.0 K 2024-11-16T08:59:11,840 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/.tmp/table/88b7aa6e22eb4a6ba5bd2f884de0de55 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/88b7aa6e22eb4a6ba5bd2f884de0de55 2024-11-16T08:59:11,841 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/88b7aa6e22eb4a6ba5bd2f884de0de55, delay=919 ms 2024-11-16T08:59:11,842 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/88b7aa6e22eb4a6ba5bd2f884de0de55, entries=2, sequenceid=13, filesize=5.3 K 2024-11-16T08:59:11,843 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false 2024-11-16T08:59:11,843 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/info/f989e87e75c14fa8bd62d443ac381f5a 2024-11-16T08:59:11,844 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/ns/7273e93eb3a2408ab3aaacb095ec0a0e 2024-11-16T08:59:11,844 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/table/88b7aa6e22eb4a6ba5bd2f884de0de55 2024-11-16T08:59:11,848 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-16T08:59:11,848 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-16T08:59:11,848 INFO [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-16T08:59:11,849 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731747551768Running coprocessor pre-close hooks at 1731747551768Disabling compacts and flushes for region at 1731747551768Disabling writes for close at 1731747551768Obtaining lock to block concurrent updates at 1731747551768Preparing flush snapshotting stores in 1588230740 at 1731747551768Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1731747551769 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731747551769Flushing 1588230740/info: creating writer at 1731747551769Flushing 1588230740/info: appending metadata at 1731747551790 (+21 ms)Flushing 1588230740/info: closing flushed file at 1731747551790Flushing 1588230740/ns: creating writer at 1731747551793 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731747551811 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731747551811Flushing 1588230740/table: creating writer at 1731747551815 (+4 ms)Flushing 1588230740/table: appending metadata at 1731747551832 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731747551832Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@292945e2: reopening flushed file at 1731747551835 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a31c64d: reopening flushed file at 1731747551837 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b09c4c2: reopening flushed file at 1731747551840 (+3 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 74ms, sequenceid=13, compaction requested=false at 1731747551843 (+3 ms)Writing region close event to WAL at 1731747551844 (+1 ms)Running coprocessor post-close hooks at 1731747551848 (+4 ms)Closed at 1731747551848 2024-11-16T08:59:11,849 DEBUG [RS_CLOSE_META-regionserver/16bf49104b8f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-16T08:59:11,910 INFO [RS:2;16bf49104b8f:42921 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T08:59:11,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:11,910 INFO [RS:2;16bf49104b8f:42921 {}] regionserver.HRegionServer(1031): Exiting; stopping=16bf49104b8f,42921,1731747550882; zookeeper connection closed. 2024-11-16T08:59:11,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42921-0x10144281b0a0003, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:11,911 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4a106651 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4a106651 2024-11-16T08:59:11,968 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(976): stopping server 16bf49104b8f,38779,1731747547431; all regions closed. 2024-11-16T08:59:11,972 DEBUG [RS:1;16bf49104b8f:38779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs 2024-11-16T08:59:11,972 INFO [RS:1;16bf49104b8f:38779 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16bf49104b8f%2C38779%2C1731747547431.meta:.meta(num 1731747548510) 2024-11-16T08:59:11,976 DEBUG [RS:1;16bf49104b8f:38779 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/oldWALs 2024-11-16T08:59:11,976 INFO [RS:1;16bf49104b8f:38779 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 16bf49104b8f%2C38779%2C1731747547431:(num 1731747548116) 2024-11-16T08:59:11,976 DEBUG [RS:1;16bf49104b8f:38779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-16T08:59:11,976 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.LeaseManager(133): Closed leases 2024-11-16T08:59:11,977 INFO [RS:1;16bf49104b8f:38779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T08:59:11,977 INFO [RS:1;16bf49104b8f:38779 {}] hbase.ChoreService(370): Chore service for: regionserver/16bf49104b8f:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-16T08:59:11,978 INFO [RS:1;16bf49104b8f:38779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T08:59:11,978 INFO [regionserver/16bf49104b8f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T08:59:11,978 INFO [RS:1;16bf49104b8f:38779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38779 2024-11-16T08:59:11,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-16T08:59:11,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/16bf49104b8f,38779,1731747547431 2024-11-16T08:59:11,998 INFO [RS:1;16bf49104b8f:38779 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T08:59:12,006 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [16bf49104b8f,38779,1731747547431] 2024-11-16T08:59:12,014 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/16bf49104b8f,38779,1731747547431 already deleted, retry=false 2024-11-16T08:59:12,015 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 16bf49104b8f,38779,1731747547431 expired; onlineServers=0 2024-11-16T08:59:12,015 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '16bf49104b8f,35741,1731747547220' ***** 2024-11-16T08:59:12,015 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-16T08:59:12,015 INFO [M:0;16bf49104b8f:35741 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-16T08:59:12,015 INFO [M:0;16bf49104b8f:35741 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-16T08:59:12,015 DEBUG [M:0;16bf49104b8f:35741 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-16T08:59:12,016 DEBUG [M:0;16bf49104b8f:35741 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-16T08:59:12,016 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-16T08:59:12,016 DEBUG [master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.large.0-1731747547848 {}] cleaner.HFileCleaner(306): Exit Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.large.0-1731747547848,5,FailOnTimeoutGroup] 2024-11-16T08:59:12,016 DEBUG [master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.small.0-1731747547849 {}] cleaner.HFileCleaner(306): Exit Thread[master/16bf49104b8f:0:becomeActiveMaster-HFileCleaner.small.0-1731747547849,5,FailOnTimeoutGroup] 2024-11-16T08:59:12,016 INFO [M:0;16bf49104b8f:35741 {}] hbase.ChoreService(370): Chore service for: master/16bf49104b8f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-16T08:59:12,016 INFO [M:0;16bf49104b8f:35741 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-16T08:59:12,017 DEBUG [M:0;16bf49104b8f:35741 {}] master.HMaster(1795): Stopping service threads 2024-11-16T08:59:12,017 INFO [M:0;16bf49104b8f:35741 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-16T08:59:12,017 INFO [M:0;16bf49104b8f:35741 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-16T08:59:12,018 INFO [M:0;16bf49104b8f:35741 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-16T08:59:12,018 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-16T08:59:12,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-16T08:59:12,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-16T08:59:12,027 DEBUG [M:0;16bf49104b8f:35741 {}] zookeeper.ZKUtil(347): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-16T08:59:12,027 WARN [M:0;16bf49104b8f:35741 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-16T08:59:12,028 INFO [M:0;16bf49104b8f:35741 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/.lastflushedseqids 2024-11-16T08:59:12,035 INFO [M:0;16bf49104b8f:35741 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-16T08:59:12,035 INFO [M:0;16bf49104b8f:35741 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-16T08:59:12,035 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-16T08:59:12,035 INFO [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:12,035 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:12,035 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-16T08:59:12,035 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:12,035 INFO [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-16T08:59:12,043 DEBUG [M:0;16bf49104b8f:35741 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2dfb2ccac35945c597cd0541958c71ec 2024-11-16T08:59:12,056 DEBUG [M:0;16bf49104b8f:35741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2dfb2ccac35945c597cd0541958c71ec is 82, key is hbase:meta,,1/info:regioninfo/1731747548542/Put/seqid=0 2024-11-16T08:59:12,057 INFO [M:0;16bf49104b8f:35741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2dfb2ccac35945c597cd0541958c71ec 2024-11-16T08:59:12,064 DEBUG [M:0;16bf49104b8f:35741 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 670b0ac00a78426db79c81e174d9b01c 2024-11-16T08:59:12,076 DEBUG [M:0;16bf49104b8f:35741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/670b0ac00a78426db79c81e174d9b01c is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731747549062/Put/seqid=0 2024-11-16T08:59:12,077 INFO [M:0;16bf49104b8f:35741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/670b0ac00a78426db79c81e174d9b01c 2024-11-16T08:59:12,083 DEBUG [M:0;16bf49104b8f:35741 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7e27351d5eec4405a839330d271ce6f9 2024-11-16T08:59:12,097 DEBUG [M:0;16bf49104b8f:35741 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e27351d5eec4405a839330d271ce6f9 is 69, key is 16bf49104b8f,38779,1731747547431/rs:state/1731747547913/Put/seqid=0 2024-11-16T08:59:12,097 INFO [M:0;16bf49104b8f:35741 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e27351d5eec4405a839330d271ce6f9 2024-11-16T08:59:12,099 INFO [M:0;16bf49104b8f:35741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7e27351d5eec4405a839330d271ce6f9 2024-11-16T08:59:12,099 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2dfb2ccac35945c597cd0541958c71ec as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2dfb2ccac35945c597cd0541958c71ec 2024-11-16T08:59:12,101 INFO [M:0;16bf49104b8f:35741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2dfb2ccac35945c597cd0541958c71ec, entries=8, sequenceid=83, filesize=5.5 K 2024-11-16T08:59:12,101 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/670b0ac00a78426db79c81e174d9b01c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/670b0ac00a78426db79c81e174d9b01c 2024-11-16T08:59:12,103 INFO [M:0;16bf49104b8f:35741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/670b0ac00a78426db79c81e174d9b01c, entries=9, sequenceid=83, filesize=6.4 K 2024-11-16T08:59:12,103 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7e27351d5eec4405a839330d271ce6f9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7e27351d5eec4405a839330d271ce6f9 2024-11-16T08:59:12,104 INFO [M:0;16bf49104b8f:35741 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7e27351d5eec4405a839330d271ce6f9 2024-11-16T08:59:12,105 INFO [M:0;16bf49104b8f:35741 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/fee2b7bf-3b54-c029-c1ff-0fcdd1b2995f/df9b032c-042a-a854-a6b2-feb84c0c61b8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7e27351d5eec4405a839330d271ce6f9, entries=3, sequenceid=83, filesize=5.3 K 2024-11-16T08:59:12,105 INFO [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=83, compaction requested=false 2024-11-16T08:59:12,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:12,106 INFO [RS:1;16bf49104b8f:38779 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T08:59:12,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38779-0x10144281b0a0002, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:12,106 INFO [RS:1;16bf49104b8f:38779 {}] regionserver.HRegionServer(1031): Exiting; stopping=16bf49104b8f,38779,1731747547431; zookeeper connection closed. 2024-11-16T08:59:12,107 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@594e357c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@594e357c 2024-11-16T08:59:12,107 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-16T08:59:12,111 INFO [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-16T08:59:12,111 DEBUG [M:0;16bf49104b8f:35741 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731747552035Disabling compacts and flushes for region at 1731747552035Disabling writes for close at 1731747552035Obtaining lock to block concurrent updates at 1731747552035Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731747552035Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1731747552036 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731747552036Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731747552036Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731747552056 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731747552056Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731747552058 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731747552076 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731747552076Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731747552079 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731747552096 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731747552096Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fce96ac: reopening flushed file at 1731747552099 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60122e39: reopening flushed file at 1731747552101 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a7e0123: reopening flushed file at 1731747552103 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 70ms, sequenceid=83, compaction requested=false at 1731747552105 (+2 ms)Writing region close event to WAL at 1731747552111 (+6 ms)Closed at 1731747552111 2024-11-16T08:59:12,112 INFO [M:0;16bf49104b8f:35741 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-16T08:59:12,112 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-16T08:59:12,112 INFO [M:0;16bf49104b8f:35741 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35741 2024-11-16T08:59:12,113 INFO [M:0;16bf49104b8f:35741 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-16T08:59:12,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:12,235 INFO [M:0;16bf49104b8f:35741 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-16T08:59:12,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35741-0x10144281b0a0000, quorum=127.0.0.1:61244, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-16T08:59:12,238 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-16T08:59:12,239 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-16T08:59:12,239 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-16T08:59:12,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=71 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=137 (was 140), ProcessCount=11 (was 11), AvailableMemoryMB=6533 (was 6613)