2024-11-23 05:54:56,968 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 05:54:56,988 main DEBUG Took 0.017379 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 05:54:56,988 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 05:54:56,989 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 05:54:56,990 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 05:54:56,991 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:56,999 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 05:54:57,016 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,018 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,019 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,019 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,020 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,020 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,022 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,022 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,023 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,023 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,025 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,025 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,026 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,027 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,028 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,028 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,029 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,029 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,030 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,030 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,031 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,031 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,032 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,033 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:54:57,033 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,034 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 05:54:57,036 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:54:57,037 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 05:54:57,039 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 05:54:57,040 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 05:54:57,042 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 05:54:57,042 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 05:54:57,050 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 05:54:57,053 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 05:54:57,054 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 05:54:57,055 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 05:54:57,055 main DEBUG createAppenders(={Console}) 2024-11-23 05:54:57,056 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-23 05:54:57,056 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-23 05:54:57,056 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-23 05:54:57,057 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 05:54:57,057 main DEBUG OutputStream closed 2024-11-23 05:54:57,058 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 05:54:57,058 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 05:54:57,058 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-23 05:54:57,133 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 05:54:57,135 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 05:54:57,136 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 05:54:57,137 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 05:54:57,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 05:54:57,138 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 05:54:57,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 05:54:57,139 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 05:54:57,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 05:54:57,140 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 05:54:57,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 05:54:57,141 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 05:54:57,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 05:54:57,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 05:54:57,142 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 05:54:57,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 05:54:57,143 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 05:54:57,144 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 05:54:57,146 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 05:54:57,146 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-23 05:54:57,146 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 05:54:57,147 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-23T05:54:57,162 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-23 05:54:57,165 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 05:54:57,165 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T05:54:57,388 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362 2024-11-23T05:54:57,414 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=365, ProcessCount=11, AvailableMemoryMB=8134 2024-11-23T05:54:57,517 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T05:54:57,604 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:57,618 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/cluster_1c40058e-7b22-e3b4-c74c-78d947dc070d, deleteOnExit=true 2024-11-23T05:54:57,742 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/cluster_1c40058e-7b22-e3b4-c74c-78d947dc070d/zookeeper_0, clientPort=64102, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/cluster_1c40058e-7b22-e3b4-c74c-78d947dc070d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/cluster_1c40058e-7b22-e3b4-c74c-78d947dc070d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T05:54:57,753 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64102 2024-11-23T05:54:57,754 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:57,827 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a with version=8 2024-11-23T05:54:57,828 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/hbase-staging 2024-11-23T05:54:57,915 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T05:54:58,197 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:54:58,205 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:58,206 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:58,214 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:54:58,214 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:58,214 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:54:58,337 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T05:54:58,412 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:54:58,449 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T05:54:58,460 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T05:54:58,463 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:54:58,502 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 93907 (auto-detected) 2024-11-23T05:54:58,504 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T05:54:58,527 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46451 2024-11-23T05:54:58,553 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46451 connecting to ZooKeeper ensemble=127.0.0.1:64102 2024-11-23T05:54:58,879 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:464510x0, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:54:58,881 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46451-0x101678c04fe0000 connected 2024-11-23T05:54:58,957 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:58,958 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:58,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:54:58,971 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a, hbase.cluster.distributed=false 2024-11-23T05:54:58,993 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:54:58,998 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46451 2024-11-23T05:54:58,999 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46451 2024-11-23T05:54:59,001 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46451 2024-11-23T05:54:59,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46451 2024-11-23T05:54:59,002 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46451 2024-11-23T05:54:59,073 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:54:59,074 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:59,075 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:59,075 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:54:59,075 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:59,075 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:54:59,078 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T05:54:59,080 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:54:59,081 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46405 2024-11-23T05:54:59,083 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46405 connecting to ZooKeeper ensemble=127.0.0.1:64102 2024-11-23T05:54:59,084 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:59,085 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:59,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:464050x0, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:54:59,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:464050x0, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:54:59,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46405-0x101678c04fe0001 connected 2024-11-23T05:54:59,110 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T05:54:59,123 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.cache 2024-11-23T05:54:59,129 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T05:54:59,130 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T05:54:59,132 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T05:54:59,134 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T05:54:59,134 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T05:54:59,141 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T05:54:59,145 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T05:54:59,145 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.persistence is complete. 2024-11-23T05:54:59,147 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T05:54:59,150 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T05:54:59,155 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:54:59,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46405 2024-11-23T05:54:59,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46405 2024-11-23T05:54:59,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46405 2024-11-23T05:54:59,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46405 2024-11-23T05:54:59,160 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46405 2024-11-23T05:54:59,178 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:54:59,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:59,178 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:59,179 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:54:59,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:54:59,179 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:54:59,179 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T05:54:59,179 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:54:59,180 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37149 2024-11-23T05:54:59,182 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37149 connecting to ZooKeeper ensemble=127.0.0.1:64102 2024-11-23T05:54:59,183 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:59,184 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:59,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:371490x0, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:54:59,212 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37149-0x101678c04fe0002 connected 2024-11-23T05:54:59,212 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:371490x0, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:54:59,213 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T05:54:59,214 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.cache 2024-11-23T05:54:59,214 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T05:54:59,214 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T05:54:59,214 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T05:54:59,215 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T05:54:59,215 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T05:54:59,215 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T05:54:59,216 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T05:54:59,216 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.persistence is complete. 2024-11-23T05:54:59,216 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T05:54:59,218 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T05:54:59,220 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:54:59,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37149 2024-11-23T05:54:59,226 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37149 2024-11-23T05:54:59,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37149 2024-11-23T05:54:59,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37149 2024-11-23T05:54:59,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37149 2024-11-23T05:54:59,250 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9abfc7c2569e:46451 2024-11-23T05:54:59,251 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9abfc7c2569e,46451,1732341298038 2024-11-23T05:54:59,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:54:59,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:54:59,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:54:59,282 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9abfc7c2569e,46451,1732341298038 2024-11-23T05:54:59,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T05:54:59,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T05:54:59,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:54:59,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:54:59,312 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:54:59,314 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T05:54:59,315 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9abfc7c2569e,46451,1732341298038 from backup master directory 2024-11-23T05:54:59,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:54:59,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9abfc7c2569e,46451,1732341298038 2024-11-23T05:54:59,328 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:54:59,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:54:59,329 WARN [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:54:59,330 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9abfc7c2569e,46451,1732341298038 2024-11-23T05:54:59,332 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T05:54:59,334 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T05:54:59,345 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/hbase.id] with ID: bebd81ab-4d70-4a34-a92b-bdfea1fae7fa 2024-11-23T05:54:59,345 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.tmp/hbase.id 2024-11-23T05:54:59,350 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/hbase.id] 2024-11-23T05:54:59,386 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:54:59,392 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T05:54:59,418 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-23T05:54:59,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:54:59,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:54:59,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:54:59,469 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:54:59,473 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T05:54:59,515 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:54:59,521 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:54:59,576 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store 2024-11-23T05:54:59,602 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T05:54:59,607 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:54:59,609 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T05:54:59,609 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:54:59,609 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:54:59,611 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T05:54:59,612 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:54:59,612 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:54:59,612 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732341299608Disabling compacts and flushes for region at 1732341299608Disabling writes for close at 1732341299611 (+3 ms)Writing region close event to WAL at 1732341299612 (+1 ms)Closed at 1732341299612 2024-11-23T05:54:59,613 WARN [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/.initializing 2024-11-23T05:54:59,613 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/WALs/9abfc7c2569e,46451,1732341298038 2024-11-23T05:54:59,627 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:54:59,643 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9abfc7c2569e%2C46451%2C1732341298038, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/WALs/9abfc7c2569e,46451,1732341298038, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/oldWALs, maxLogs=10 2024-11-23T05:54:59,683 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/WALs/9abfc7c2569e,46451,1732341298038/9abfc7c2569e%2C46451%2C1732341298038.1732341299649 2024-11-23T05:54:59,684 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:54:59,684 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:54:59,685 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:54:59,688 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,689 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T05:54:59,756 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:54:59,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:54:59,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T05:54:59,763 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:54:59,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:54:59,764 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,767 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T05:54:59,768 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:54:59,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:54:59,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,772 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T05:54:59,773 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:54:59,773 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:54:59,773 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,778 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,779 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,783 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,784 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,787 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T05:54:59,790 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:54:59,797 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:54:59,797 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60981875, jitterRate=-0.09129925072193146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T05:54:59,805 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732341299701Initializing all the Stores at 1732341299702 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341299702Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341299703 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341299703Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341299703Cleaning up temporary data from old regions at 1732341299784 (+81 ms)Region opened successfully at 1732341299805 (+21 ms) 2024-11-23T05:54:59,806 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T05:54:59,834 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9fc033, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:54:59,864 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T05:54:59,876 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T05:54:59,876 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T05:54:59,879 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T05:54:59,880 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T05:54:59,885 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-23T05:54:59,886 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T05:54:59,909 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T05:54:59,919 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T05:54:59,978 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T05:54:59,982 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T05:54:59,985 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T05:54:59,994 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T05:54:59,996 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T05:55:00,000 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T05:55:00,011 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T05:55:00,013 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T05:55:00,019 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T05:55:00,040 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T05:55:00,052 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T05:55:00,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:00,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:00,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:00,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,082 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9abfc7c2569e,46451,1732341298038, sessionid=0x101678c04fe0000, setting cluster-up flag (Was=false) 2024-11-23T05:55:00,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,136 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T05:55:00,139 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9abfc7c2569e,46451,1732341298038 2024-11-23T05:55:00,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,162 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,186 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T05:55:00,191 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9abfc7c2569e,46451,1732341298038 2024-11-23T05:55:00,198 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T05:55:00,234 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(746): ClusterId : bebd81ab-4d70-4a34-a92b-bdfea1fae7fa 2024-11-23T05:55:00,234 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(746): ClusterId : bebd81ab-4d70-4a34-a92b-bdfea1fae7fa 2024-11-23T05:55:00,236 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T05:55:00,236 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T05:55:00,263 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T05:55:00,263 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T05:55:00,263 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T05:55:00,263 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T05:55:00,269 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:00,271 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T05:55:00,271 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T05:55:00,272 DEBUG [RS:0;9abfc7c2569e:46405 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2a9f70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:55:00,272 DEBUG [RS:1;9abfc7c2569e:37149 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b1bbcdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:55:00,281 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T05:55:00,286 DEBUG [RS:0;9abfc7c2569e:46405 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9abfc7c2569e:46405 2024-11-23T05:55:00,288 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9abfc7c2569e:37149 2024-11-23T05:55:00,288 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T05:55:00,290 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T05:55:00,290 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T05:55:00,290 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T05:55:00,290 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T05:55:00,291 DEBUG [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T05:55:00,291 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T05:55:00,294 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(2659): reportForDuty to master=9abfc7c2569e,46451,1732341298038 with port=37149, startcode=1732341299177 2024-11-23T05:55:00,294 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(2659): reportForDuty to master=9abfc7c2569e,46451,1732341298038 with port=46405, startcode=1732341299042 2024-11-23T05:55:00,295 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9abfc7c2569e,46451,1732341298038 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T05:55:00,303 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:00,304 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:00,304 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:00,304 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:00,304 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9abfc7c2569e:0, corePoolSize=10, maxPoolSize=10 2024-11-23T05:55:00,304 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,305 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:55:00,305 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,307 DEBUG [RS:1;9abfc7c2569e:37149 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T05:55:00,307 DEBUG [RS:0;9abfc7c2569e:46405 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T05:55:00,310 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:00,310 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732341330310 2024-11-23T05:55:00,311 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T05:55:00,312 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T05:55:00,314 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T05:55:00,316 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:00,316 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T05:55:00,318 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T05:55:00,318 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T05:55:00,318 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T05:55:00,319 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T05:55:00,320 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,323 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T05:55:00,324 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T05:55:00,325 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T05:55:00,327 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T05:55:00,328 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T05:55:00,330 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732341300329,5,FailOnTimeoutGroup] 2024-11-23T05:55:00,330 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732341300330,5,FailOnTimeoutGroup] 2024-11-23T05:55:00,330 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,331 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T05:55:00,331 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,332 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,341 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T05:55:00,342 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:00,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:00,353 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T05:55:00,359 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T05:55:00,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:00,360 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:00,361 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T05:55:00,369 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T05:55:00,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:00,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:00,370 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T05:55:00,376 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T05:55:00,376 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:00,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:00,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T05:55:00,381 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36835, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T05:55:00,381 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39149, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T05:55:00,384 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T05:55:00,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:00,384 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:00,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T05:55:00,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:00,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:00,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T05:55:00,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T05:55:00,387 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T05:55:00,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T05:55:00,389 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46451 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:00,392 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46451 {}] master.ServerManager(517): Registering regionserver=9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:00,393 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:55:00,393 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69423312, jitterRate=0.03448796272277832}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T05:55:00,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732341300352Initializing all the Stores at 1732341300353 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341300353Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341300353Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341300353Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341300353Cleaning up temporary data from old regions at 1732341300386 (+33 ms)Region opened successfully at 1732341300398 (+12 ms) 2024-11-23T05:55:00,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T05:55:00,398 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T05:55:00,398 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T05:55:00,399 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T05:55:00,399 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T05:55:00,401 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T05:55:00,402 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732341300398Disabling compacts and flushes for region at 1732341300398Disabling writes for close at 1732341300399 (+1 ms)Writing region close event to WAL at 1732341300401 (+2 ms)Closed at 1732341300401 2024-11-23T05:55:00,405 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:00,405 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T05:55:00,406 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46451 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:00,406 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46451 {}] master.ServerManager(517): Registering regionserver=9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:00,411 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:00,411 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T05:55:00,411 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T05:55:00,413 DEBUG [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:00,414 DEBUG [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T05:55:00,414 DEBUG [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T05:55:00,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T05:55:00,425 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T05:55:00,428 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T05:55:00,436 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:00,469 DEBUG [RS:1;9abfc7c2569e:37149 {}] zookeeper.ZKUtil(111): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:00,469 WARN [RS:1;9abfc7c2569e:37149 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:55:00,470 DEBUG [RS:0;9abfc7c2569e:46405 {}] zookeeper.ZKUtil(111): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:00,470 INFO [RS:1;9abfc7c2569e:37149 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:00,470 WARN [RS:0;9abfc7c2569e:46405 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:55:00,470 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:00,470 INFO [RS:0;9abfc7c2569e:46405 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:00,470 DEBUG [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:00,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9abfc7c2569e,46405,1732341299042] 2024-11-23T05:55:00,471 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9abfc7c2569e,37149,1732341299177] 2024-11-23T05:55:00,499 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T05:55:00,499 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T05:55:00,517 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T05:55:00,517 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T05:55:00,521 INFO [RS:0;9abfc7c2569e:46405 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T05:55:00,521 INFO [RS:1;9abfc7c2569e:37149 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T05:55:00,521 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,521 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,522 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T05:55:00,525 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T05:55:00,527 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T05:55:00,527 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T05:55:00,529 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,529 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,529 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,529 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,529 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,529 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,529 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,529 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,529 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:55:00,530 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:55:00,530 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,530 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,531 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,531 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,531 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:00,531 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,531 DEBUG [RS:0;9abfc7c2569e:46405 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:00,531 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,531 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:00,531 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:00,531 DEBUG [RS:1;9abfc7c2569e:37149 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:00,532 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,532 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,532 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,532 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,533 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,533 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46405,1732341299042-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:55:00,535 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,535 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,535 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,535 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,535 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,535 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,37149,1732341299177-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:55:00,554 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T05:55:00,556 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,37149,1732341299177-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,557 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T05:55:00,557 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,557 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.Replication(171): 9abfc7c2569e,37149,1732341299177 started 2024-11-23T05:55:00,557 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46405,1732341299042-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,557 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,557 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.Replication(171): 9abfc7c2569e,46405,1732341299042 started 2024-11-23T05:55:00,575 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,576 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1482): Serving as 9abfc7c2569e,37149,1732341299177, RpcServer on 9abfc7c2569e/172.17.0.2:37149, sessionid=0x101678c04fe0002 2024-11-23T05:55:00,576 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T05:55:00,577 DEBUG [RS:1;9abfc7c2569e:37149 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:00,577 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,37149,1732341299177' 2024-11-23T05:55:00,577 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T05:55:00,578 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T05:55:00,579 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T05:55:00,579 WARN [9abfc7c2569e:46451 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T05:55:00,579 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T05:55:00,579 DEBUG [RS:1;9abfc7c2569e:37149 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:00,579 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,37149,1732341299177' 2024-11-23T05:55:00,579 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T05:55:00,580 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T05:55:00,581 DEBUG [RS:1;9abfc7c2569e:37149 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T05:55:00,581 INFO [RS:1;9abfc7c2569e:37149 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T05:55:00,581 INFO [RS:1;9abfc7c2569e:37149 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T05:55:00,581 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:00,581 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(1482): Serving as 9abfc7c2569e,46405,1732341299042, RpcServer on 9abfc7c2569e/172.17.0.2:46405, sessionid=0x101678c04fe0001 2024-11-23T05:55:00,582 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T05:55:00,582 DEBUG [RS:0;9abfc7c2569e:46405 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:00,582 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,46405,1732341299042' 2024-11-23T05:55:00,582 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T05:55:00,583 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T05:55:00,584 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T05:55:00,584 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T05:55:00,584 DEBUG [RS:0;9abfc7c2569e:46405 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:00,584 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,46405,1732341299042' 2024-11-23T05:55:00,584 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T05:55:00,585 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T05:55:00,585 DEBUG [RS:0;9abfc7c2569e:46405 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T05:55:00,585 INFO [RS:0;9abfc7c2569e:46405 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T05:55:00,586 INFO [RS:0;9abfc7c2569e:46405 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T05:55:00,686 INFO [RS:1;9abfc7c2569e:37149 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:55:00,686 INFO [RS:0;9abfc7c2569e:46405 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:55:00,687 INFO [RS:1;9abfc7c2569e:37149 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9abfc7c2569e%2C37149%2C1732341299177, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,37149,1732341299177, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs, maxLogs=55 2024-11-23T05:55:00,687 INFO [RS:0;9abfc7c2569e:46405 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9abfc7c2569e%2C46405%2C1732341299042, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,46405,1732341299042, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs, maxLogs=55 2024-11-23T05:55:00,708 INFO [RS:1;9abfc7c2569e:37149 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,37149,1732341299177/9abfc7c2569e%2C37149%2C1732341299177.1732341300691 2024-11-23T05:55:00,708 INFO [RS:0;9abfc7c2569e:46405 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,46405,1732341299042/9abfc7c2569e%2C46405%2C1732341299042.1732341300691 2024-11-23T05:55:00,708 DEBUG [RS:1;9abfc7c2569e:37149 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:00,711 DEBUG [RS:0;9abfc7c2569e:46405 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:00,832 DEBUG [9abfc7c2569e:46451 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T05:55:00,844 DEBUG [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(204): Hosts are {9abfc7c2569e=0} racks are {/default-rack=0} 2024-11-23T05:55:00,847 DEBUG [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T05:55:00,847 DEBUG [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T05:55:00,847 DEBUG [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T05:55:00,847 DEBUG [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T05:55:00,847 INFO [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T05:55:00,847 INFO [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T05:55:00,847 DEBUG [9abfc7c2569e:46451 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T05:55:00,853 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:00,857 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9abfc7c2569e,37149,1732341299177, state=OPENING 2024-11-23T05:55:00,894 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T05:55:00,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:00,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:00,904 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:00,905 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:00,908 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T05:55:00,912 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9abfc7c2569e,37149,1732341299177}] 2024-11-23T05:55:01,086 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T05:55:01,089 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60663, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T05:55:01,096 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T05:55:01,097 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:01,097 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T05:55:01,099 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9abfc7c2569e%2C37149%2C1732341299177.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,37149,1732341299177, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs, maxLogs=55 2024-11-23T05:55:01,117 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,37149,1732341299177/9abfc7c2569e%2C37149%2C1732341299177.meta.1732341301101.meta 2024-11-23T05:55:01,118 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:01,118 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:01,120 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T05:55:01,122 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T05:55:01,126 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T05:55:01,129 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T05:55:01,130 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:01,130 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T05:55:01,130 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T05:55:01,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T05:55:01,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T05:55:01,132 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:01,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:01,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T05:55:01,133 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T05:55:01,134 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:01,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:01,134 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T05:55:01,135 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T05:55:01,135 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:01,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:01,135 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T05:55:01,137 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T05:55:01,137 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:01,137 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:01,137 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T05:55:01,138 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:01,139 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:01,140 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T05:55:01,140 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T05:55:01,140 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T05:55:01,141 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T05:55:01,141 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66883780, jitterRate=-0.003354012966156006}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T05:55:01,142 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T05:55:01,143 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732341301130Writing region info on filesystem at 1732341301131 (+1 ms)Initializing all the Stores at 1732341301131Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341301131Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341301131Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341301131Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341301131Cleaning up temporary data from old regions at 1732341301140 (+9 ms)Running coprocessor post-open hooks at 1732341301142 (+2 ms)Region opened successfully at 1732341301143 (+1 ms) 2024-11-23T05:55:01,149 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732341301077 2024-11-23T05:55:01,158 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T05:55:01,158 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T05:55:01,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:01,160 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9abfc7c2569e,37149,1732341299177, state=OPEN 2024-11-23T05:55:01,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:55:01,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:55:01,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:55:01,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:01,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:01,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:01,195 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:01,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T05:55:01,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9abfc7c2569e,37149,1732341299177 in 284 msec 2024-11-23T05:55:01,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T05:55:01,206 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 788 msec 2024-11-23T05:55:01,207 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:01,207 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T05:55:01,224 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:55:01,225 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,37149,1732341299177, seqNum=-1] 2024-11-23T05:55:01,243 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:55:01,245 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32865, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:55:01,262 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0340 sec 2024-11-23T05:55:01,262 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732341301262, completionTime=-1 2024-11-23T05:55:01,265 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T05:55:01,265 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T05:55:01,289 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T05:55:01,289 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732341361289 2024-11-23T05:55:01,289 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732341421289 2024-11-23T05:55:01,289 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-11-23T05:55:01,291 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T05:55:01,297 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46451,1732341298038-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:01,297 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46451,1732341298038-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:01,297 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46451,1732341298038-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:01,299 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9abfc7c2569e:46451, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:01,299 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:01,300 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:01,305 DEBUG [master/9abfc7c2569e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T05:55:01,329 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.999sec 2024-11-23T05:55:01,330 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T05:55:01,331 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T05:55:01,332 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T05:55:01,333 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T05:55:01,333 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T05:55:01,333 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46451,1732341298038-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:55:01,334 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46451,1732341298038-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T05:55:01,335 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T05:55:01,336 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T05:55:01,336 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,46451,1732341298038-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:01,343 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598889d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:55:01,345 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T05:55:01,345 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T05:55:01,348 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,46451,-1 for getting cluster id 2024-11-23T05:55:01,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:55:01,359 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bebd81ab-4d70-4a34-a92b-bdfea1fae7fa' 2024-11-23T05:55:01,362 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:55:01,362 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bebd81ab-4d70-4a34-a92b-bdfea1fae7fa" 2024-11-23T05:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e066dbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:55:01,364 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,46451,-1] 2024-11-23T05:55:01,367 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:55:01,368 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:01,369 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:55:01,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46a17ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:55:01,373 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:55:01,380 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,37149,1732341299177, seqNum=-1] 2024-11-23T05:55:01,380 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:55:01,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:55:01,415 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9abfc7c2569e,46451,1732341298038 2024-11-23T05:55:01,423 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:55:01,428 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,46451,1732341298038 2024-11-23T05:55:01,430 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@333e904f 2024-11-23T05:55:01,431 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:55:01,434 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:55:01,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1732341301415', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:55:01,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 2024-11-23T05:55:01,450 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:55:01,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1732341301415" procId is: 4 2024-11-23T05:55:01,453 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:01,455 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:55:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:01,469 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45fc5f2be9963024234422d7705a81ae, NAME => 'testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1732341301415', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:01,475 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:01,476 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(1722): Closing 45fc5f2be9963024234422d7705a81ae, disabling compactions & flushes 2024-11-23T05:55:01,476 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:01,476 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:01,476 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. after waiting 0 ms 2024-11-23T05:55:01,476 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:01,476 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:01,476 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1732341301415-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45fc5f2be9963024234422d7705a81ae: Waiting for close lock at 1732341301476Disabling compacts and flushes for region at 1732341301476Disabling writes for close at 1732341301476Writing region close event to WAL at 1732341301476Closed at 1732341301476 2024-11-23T05:55:01,478 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:55:01,483 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732341301478"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732341301478"}]},"ts":"1732341301478"} 2024-11-23T05:55:01,489 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:55:01,491 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:55:01,493 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732341301415","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732341301491"}]},"ts":"1732341301491"} 2024-11-23T05:55:01,498 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732341301415, state=ENABLING in hbase:meta 2024-11-23T05:55:01,499 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9abfc7c2569e=0} racks are {/default-rack=0} 2024-11-23T05:55:01,500 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T05:55:01,500 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T05:55:01,500 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T05:55:01,500 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T05:55:01,500 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T05:55:01,500 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T05:55:01,500 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T05:55:01,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, ASSIGN}] 2024-11-23T05:55:01,503 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, ASSIGN 2024-11-23T05:55:01,504 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, ASSIGN; state=OFFLINE, location=9abfc7c2569e,46405,1732341299042; forceNewPlan=false, retain=false 2024-11-23T05:55:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:01,658 INFO [9abfc7c2569e:46451 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T05:55:01,659 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=45fc5f2be9963024234422d7705a81ae, regionState=OPENING, regionLocation=9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:01,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, ASSIGN because future has completed 2024-11-23T05:55:01,666 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,46405,1732341299042}] 2024-11-23T05:55:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:01,821 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T05:55:01,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55513, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T05:55:01,829 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:01,830 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 45fc5f2be9963024234422d7705a81ae, NAME => 'testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:01,831 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732341301415 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,831 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:01,832 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,832 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,834 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,839 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fc5f2be9963024234422d7705a81ae columnFamilyName family 2024-11-23T05:55:01,839 DEBUG [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:01,840 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.HStore(327): Store=45fc5f2be9963024234422d7705a81ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:01,840 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,842 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,842 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,842 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,842 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,843 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,849 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:55:01,850 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 45fc5f2be9963024234422d7705a81ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58958432, jitterRate=-0.12145090103149414}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T05:55:01,850 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:01,851 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 45fc5f2be9963024234422d7705a81ae: Running coprocessor pre-open hook at 1732341301833Writing region info on filesystem at 1732341301833Initializing all the Stores at 1732341301833Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341301833Cleaning up temporary data from old regions at 1732341301842 (+9 ms)Running coprocessor post-open hooks at 1732341301850 (+8 ms)Region opened successfully at 1732341301851 (+1 ms) 2024-11-23T05:55:01,852 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae., pid=6, masterSystemTime=1732341301820 2024-11-23T05:55:01,856 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:01,856 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:01,856 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=45fc5f2be9963024234422d7705a81ae, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:01,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,46405,1732341299042 because future has completed 2024-11-23T05:55:01,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T05:55:01,865 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,46405,1732341299042 in 196 msec 2024-11-23T05:55:01,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T05:55:01,868 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, ASSIGN in 364 msec 2024-11-23T05:55:01,869 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:55:01,870 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1732341301415","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732341301869"}]},"ts":"1732341301869"} 2024-11-23T05:55:01,872 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1732341301415, state=ENABLED in hbase:meta 2024-11-23T05:55:01,874 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:55:01,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1732341301415 in 430 msec 2024-11-23T05:55:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:02,088 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1732341301415 get assigned. Timeout = 60000ms 2024-11-23T05:55:02,088 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1732341301415 completed 2024-11-23T05:55:02,090 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T05:55:02,095 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1732341301415 assigned to meta. Checking AM states. 2024-11-23T05:55:02,096 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T05:55:02,096 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1732341301415 assigned. 2024-11-23T05:55:02,103 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1732341301415', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae., hostname=9abfc7c2569e,46405,1732341299042, seqNum=2] 2024-11-23T05:55:02,104 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:55:02,107 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:55:02,117 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 45fc5f2be9963024234422d7705a81ae 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T05:55:02,143 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 054d208947624ee08161e6816a1c6734 2024-11-23T05:55:02,183 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/.tmp/family/054d208947624ee08161e6816a1c6734 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T05:55:02,188 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/.tmp/family/054d208947624ee08161e6816a1c6734 2024-11-23T05:55:02,217 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/.tmp/family/054d208947624ee08161e6816a1c6734 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734 2024-11-23T05:55:02,219 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734, delay=953 ms 2024-11-23T05:55:02,220 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T05:55:02,227 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 45fc5f2be9963024234422d7705a81ae in 109ms, sequenceid=6, compaction requested=false 2024-11-23T05:55:02,228 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1732341301415' 2024-11-23T05:55:02,229 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 45fc5f2be9963024234422d7705a81ae: 2024-11-23T05:55:03,176 DEBUG [hfile-prefetch-1732341302219 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 054d208947624ee08161e6816a1c6734_0. 2024-11-23T05:55:03,181 DEBUG [hfile-prefetch-1732341302219 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 054d208947624ee08161e6816a1c6734_117. 2024-11-23T05:55:03,182 DEBUG [hfile-prefetch-1732341302219 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734, with total blocks 2, and data blocks 1 2024-11-23T05:55:03,182 DEBUG [hfile-prefetch-1732341302219 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T05:55:03,182 DEBUG [hfile-prefetch-1732341302219 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T05:55:03,183 DEBUG [hfile-prefetch-1732341302219 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734. Will sleep 100ms and try the verification again. 2024-11-23T05:55:03,244 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=45fc5f2be9963024234422d7705a81ae, source=9abfc7c2569e,46405,1732341299042, destination=9abfc7c2569e,37149,1732341299177, warming up region on 9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:03,246 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=45fc5f2be9963024234422d7705a81ae, source=9abfc7c2569e,46405,1732341299042, destination=9abfc7c2569e,37149,1732341299177, running balancer 2024-11-23T05:55:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, REOPEN/MOVE 2024-11-23T05:55:03,248 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, REOPEN/MOVE 2024-11-23T05:55:03,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(7855): Warmup {ENCODED => 45fc5f2be9963024234422d7705a81ae, NAME => 'testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:03,249 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,250 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=45fc5f2be9963024234422d7705a81ae, regionState=CLOSING, regionLocation=9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:03,250 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fc5f2be9963024234422d7705a81ae columnFamilyName family 2024-11-23T05:55:03,250 DEBUG [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:03,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, REOPEN/MOVE because future has completed 2024-11-23T05:55:03,253 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-23T05:55:03,254 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,46405,1732341299042}] 2024-11-23T05:55:03,254 DEBUG [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734 2024-11-23T05:55:03,254 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.HStore(327): Store=45fc5f2be9963024234422d7705a81ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(1722): Closing 45fc5f2be9963024234422d7705a81ae, disabling compactions & flushes 2024-11-23T05:55:03,254 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. after waiting 0 ms 2024-11-23T05:55:03,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,256 DEBUG [StoreFileCloser-45fc5f2be9963024234422d7705a81ae-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734 2024-11-23T05:55:03,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37149 {}] regionserver.HRegion(1676): Region close journal for 45fc5f2be9963024234422d7705a81ae: Waiting for close lock at 1732341303254Disabling compacts and flushes for region at 1732341303254Disabling writes for close at 1732341303255 (+1 ms)Writing region close event to WAL at 1732341303256 (+1 ms)Closed at 1732341303256 2024-11-23T05:55:03,258 WARN [hfile-prefetch-1732341302219 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-23T05:55:03,259 DEBUG [hfile-prefetch-1732341302219 {}] hfile.PrefetchExecutor(121): Prefetch completed for 054d208947624ee08161e6816a1c6734 2024-11-23T05:55:03,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46451 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-23T05:55:03,318 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T05:55:03,318 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T05:55:03,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:03,319 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:03,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T05:55:03,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732341303318 2024-11-23T05:55:03,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T05:55:03,336 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T05:55:03,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T05:55:03,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:03,336 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:03,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-23T05:55:03,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732341303336 2024-11-23T05:55:03,343 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T05:55:03,413 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,413 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-23T05:55:03,414 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 45fc5f2be9963024234422d7705a81ae, disabling compactions & flushes 2024-11-23T05:55:03,414 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,414 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,414 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. after waiting 0 ms 2024-11-23T05:55:03,414 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,416 DEBUG [StoreFileCloser-45fc5f2be9963024234422d7705a81ae-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 054d208947624ee08161e6816a1c6734, starting offset: 0, end offset: 9223372036854775807 2024-11-23T05:55:03,418 DEBUG [StoreFileCloser-45fc5f2be9963024234422d7705a81ae-family-1 {}] bucket.BucketCache(856): removed key 054d208947624ee08161e6816a1c6734_0 from back map with offset lock 2094080 in the evict process 2024-11-23T05:55:03,425 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T05:55:03,427 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,427 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 45fc5f2be9963024234422d7705a81ae: Waiting for close lock at 1732341303414Running coprocessor pre-close hooks at 1732341303414Disabling compacts and flushes for region at 1732341303414Disabling writes for close at 1732341303414Writing region close event to WAL at 1732341303419 (+5 ms)Running coprocessor post-close hooks at 1732341303426 (+7 ms)Closed at 1732341303427 (+1 ms) 2024-11-23T05:55:03,427 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 45fc5f2be9963024234422d7705a81ae move to 9abfc7c2569e,37149,1732341299177 record at close sequenceid=6 2024-11-23T05:55:03,432 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,432 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=45fc5f2be9963024234422d7705a81ae, regionState=CLOSED 2024-11-23T05:55:03,435 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,46405,1732341299042 because future has completed 2024-11-23T05:55:03,436 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T05:55:03,436 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T05:55:03,436 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:03,436 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:03,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T05:55:03,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,46405,1732341299042 in 183 msec 2024-11-23T05:55:03,440 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T05:55:03,440 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732341303436 2024-11-23T05:55:03,440 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T05:55:03,441 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, REOPEN/MOVE; state=CLOSED, location=9abfc7c2569e,37149,1732341299177; forceNewPlan=false, retain=false 2024-11-23T05:55:03,591 INFO [9abfc7c2569e:46451 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T05:55:03,592 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=45fc5f2be9963024234422d7705a81ae, regionState=OPENING, regionLocation=9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:03,596 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, REOPEN/MOVE because future has completed 2024-11-23T05:55:03,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,37149,1732341299177}] 2024-11-23T05:55:03,757 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,758 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 45fc5f2be9963024234422d7705a81ae, NAME => 'testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:03,759 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1732341301415 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,759 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:03,759 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,759 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,760 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,762 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45fc5f2be9963024234422d7705a81ae columnFamilyName family 2024-11-23T05:55:03,762 DEBUG [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:03,764 DEBUG [StoreFileOpener-45fc5f2be9963024234422d7705a81ae-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734, delay=908 ms 2024-11-23T05:55:03,765 DEBUG [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734 2024-11-23T05:55:03,765 INFO [StoreOpener-45fc5f2be9963024234422d7705a81ae-1 {}] regionserver.HStore(327): Store=45fc5f2be9963024234422d7705a81ae/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:03,766 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,767 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,767 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,767 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,767 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,772 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,772 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 45fc5f2be9963024234422d7705a81ae; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61335588, jitterRate=-0.08602851629257202}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T05:55:03,773 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,774 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 45fc5f2be9963024234422d7705a81ae: Running coprocessor pre-open hook at 1732341303759Writing region info on filesystem at 1732341303759Initializing all the Stores at 1732341303760 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341303760Cleaning up temporary data from old regions at 1732341303767 (+7 ms)Running coprocessor post-open hooks at 1732341303773 (+6 ms)Region opened successfully at 1732341303773 2024-11-23T05:55:03,774 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae., pid=9, masterSystemTime=1732341303750 2024-11-23T05:55:03,777 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,777 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,777 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=45fc5f2be9963024234422d7705a81ae, regionState=OPEN, openSeqNum=10, regionLocation=9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:03,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,37149,1732341299177 because future has completed 2024-11-23T05:55:03,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-23T05:55:03,784 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 45fc5f2be9963024234422d7705a81ae, server=9abfc7c2569e,37149,1732341299177 in 185 msec 2024-11-23T05:55:03,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1732341301415, region=45fc5f2be9963024234422d7705a81ae, REOPEN/MOVE in 538 msec 2024-11-23T05:55:03,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T05:55:03,801 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:55:03,801 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:03,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:03,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:03,805 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:55:03,805 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T05:55:03,806 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2143720667, stopped=false 2024-11-23T05:55:03,806 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9abfc7c2569e,46451,1732341298038 2024-11-23T05:55:03,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:03,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:03,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:03,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:03,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:03,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:03,837 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T05:55:03,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:03,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:03,838 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:55:03,838 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:03,838 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:03,838 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:03,839 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9abfc7c2569e,46405,1732341299042' ***** 2024-11-23T05:55:03,839 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T05:55:03,840 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9abfc7c2569e,37149,1732341299177' ***** 2024-11-23T05:55:03,840 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T05:55:03,840 INFO [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T05:55:03,840 INFO [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T05:55:03,841 INFO [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.persistence 2024-11-23T05:55:03,841 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T05:55:03,841 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T05:55:03,841 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T05:55:03,841 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T05:55:03,841 INFO [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/bucket.persistence 2024-11-23T05:55:03,841 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T05:55:03,841 INFO [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T05:55:03,842 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T05:55:03,842 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T05:55:03,842 INFO [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T05:55:03,842 DEBUG [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1487): Thread RS:0;9abfc7c2569e:46405 started persisting bucket cache to file 2024-11-23T05:55:03,842 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T05:55:03,842 DEBUG [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:03,842 DEBUG [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:03,846 DEBUG [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T05:55:03,846 DEBUG [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732341303842 2024-11-23T05:55:03,846 DEBUG [RS:0;9abfc7c2569e:46405 {}] bucket.BucketCache(1509): Thread RS:0;9abfc7c2569e:46405 finished persisting bucket cache to file, renaming 2024-11-23T05:55:03,846 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T05:55:03,847 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T05:55:03,847 INFO [RS:0;9abfc7c2569e:46405 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T05:55:03,847 INFO [RS:0;9abfc7c2569e:46405 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T05:55:03,847 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(959): stopping server 9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:03,847 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:55:03,847 INFO [RS:0;9abfc7c2569e:46405 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9abfc7c2569e:46405. 2024-11-23T05:55:03,847 DEBUG [RS:0;9abfc7c2569e:46405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:03,847 DEBUG [RS:0;9abfc7c2569e:46405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:03,847 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(976): stopping server 9abfc7c2569e,46405,1732341299042; all regions closed. 2024-11-23T05:55:03,851 DEBUG [RS:0;9abfc7c2569e:46405 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C46405%2C1732341299042:(num 1732341300691) 2024-11-23T05:55:03,852 DEBUG [RS:0;9abfc7c2569e:46405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:55:03,852 DEBUG [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1487): Thread RS:1;9abfc7c2569e:37149 started persisting bucket cache to file 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.ChoreService(370): Chore service for: regionserver/9abfc7c2569e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T05:55:03,852 DEBUG [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:03,852 DEBUG [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T05:55:03,852 INFO [regionserver/9abfc7c2569e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T05:55:03,852 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:55:03,853 INFO [RS:0;9abfc7c2569e:46405 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46405 2024-11-23T05:55:03,855 DEBUG [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T05:55:03,856 DEBUG [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732341303852 2024-11-23T05:55:03,856 DEBUG [RS:1;9abfc7c2569e:37149 {}] bucket.BucketCache(1509): Thread RS:1;9abfc7c2569e:37149 finished persisting bucket cache to file, renaming 2024-11-23T05:55:03,856 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T05:55:03,856 INFO [RS:1;9abfc7c2569e:37149 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T05:55:03,856 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T05:55:03,856 INFO [RS:1;9abfc7c2569e:37149 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T05:55:03,856 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(3091): Received CLOSE for 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,857 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(959): stopping server 9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:03,857 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:55:03,857 INFO [RS:1;9abfc7c2569e:37149 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9abfc7c2569e:37149. 2024-11-23T05:55:03,857 DEBUG [RS:1;9abfc7c2569e:37149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:03,857 DEBUG [RS:1;9abfc7c2569e:37149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:03,857 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T05:55:03,857 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T05:55:03,857 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T05:55:03,857 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T05:55:03,857 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T05:55:03,857 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45fc5f2be9963024234422d7705a81ae, disabling compactions & flushes 2024-11-23T05:55:03,857 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,857 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,857 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. after waiting 0 ms 2024-11-23T05:55:03,857 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,858 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T05:55:03,858 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T05:55:03,858 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1325): Online Regions={45fc5f2be9963024234422d7705a81ae=testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae., 1588230740=hbase:meta,,1.1588230740} 2024-11-23T05:55:03,858 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T05:55:03,858 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T05:55:03,858 DEBUG [StoreFileCloser-45fc5f2be9963024234422d7705a81ae-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/family/054d208947624ee08161e6816a1c6734 2024-11-23T05:55:03,858 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T05:55:03,858 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T05:55:03,858 DEBUG [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 45fc5f2be9963024234422d7705a81ae 2024-11-23T05:55:03,858 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T05:55:03,858 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-23T05:55:03,863 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnRegionMove1732341301415/45fc5f2be9963024234422d7705a81ae/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-23T05:55:03,863 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,863 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45fc5f2be9963024234422d7705a81ae: Waiting for close lock at 1732341303857Running coprocessor pre-close hooks at 1732341303857Disabling compacts and flushes for region at 1732341303857Disabling writes for close at 1732341303857Writing region close event to WAL at 1732341303858 (+1 ms)Running coprocessor post-close hooks at 1732341303863 (+5 ms)Closed at 1732341303863 2024-11-23T05:55:03,863 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae. 2024-11-23T05:55:03,867 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b5ad71ed15a6424e83c0c2b775a703a9 2024-11-23T05:55:03,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:03,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9abfc7c2569e,46405,1732341299042 2024-11-23T05:55:03,877 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:55:03,878 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9abfc7c2569e,46405,1732341299042] 2024-11-23T05:55:03,886 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/info/b5ad71ed15a6424e83c0c2b775a703a9 is 205, key is testBlockEvictionOnRegionMove1732341301415,,1732341301436.45fc5f2be9963024234422d7705a81ae./info:regioninfo/1732341303777/Put/seqid=0 2024-11-23T05:55:03,887 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/info/b5ad71ed15a6424e83c0c2b775a703a9 2024-11-23T05:55:03,894 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9abfc7c2569e,46405,1732341299042 already deleted, retry=false 2024-11-23T05:55:03,894 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9abfc7c2569e,46405,1732341299042 expired; onlineServers=1 2024-11-23T05:55:03,897 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on c5effd87a2e64c6e913c88954bee422c 2024-11-23T05:55:03,911 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/ns/c5effd87a2e64c6e913c88954bee422c is 43, key is default/ns:d/1732341301251/Put/seqid=0 2024-11-23T05:55:03,912 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/ns/c5effd87a2e64c6e913c88954bee422c 2024-11-23T05:55:03,922 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 2c07fde6fdb84e719be04e043dc76fee 2024-11-23T05:55:03,936 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/table/2c07fde6fdb84e719be04e043dc76fee is 78, key is testBlockEvictionOnRegionMove1732341301415/table:state/1732341301869/Put/seqid=0 2024-11-23T05:55:03,937 INFO [regionserver/9abfc7c2569e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:03,937 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/table/2c07fde6fdb84e719be04e043dc76fee 2024-11-23T05:55:03,939 INFO [regionserver/9abfc7c2569e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:03,940 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/info/b5ad71ed15a6424e83c0c2b775a703a9 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/b5ad71ed15a6424e83c0c2b775a703a9 2024-11-23T05:55:03,941 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/b5ad71ed15a6424e83c0c2b775a703a9, delay=927 ms 2024-11-23T05:55:03,942 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/b5ad71ed15a6424e83c0c2b775a703a9, entries=15, sequenceid=15, filesize=7.6 K 2024-11-23T05:55:03,943 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/ns/c5effd87a2e64c6e913c88954bee422c as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/c5effd87a2e64c6e913c88954bee422c 2024-11-23T05:55:03,944 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/c5effd87a2e64c6e913c88954bee422c, delay=984 ms 2024-11-23T05:55:03,945 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/c5effd87a2e64c6e913c88954bee422c, entries=2, sequenceid=15, filesize=5.0 K 2024-11-23T05:55:03,946 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/table/2c07fde6fdb84e719be04e043dc76fee as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/2c07fde6fdb84e719be04e043dc76fee 2024-11-23T05:55:03,947 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/2c07fde6fdb84e719be04e043dc76fee, delay=939 ms 2024-11-23T05:55:03,948 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/2c07fde6fdb84e719be04e043dc76fee, entries=2, sequenceid=15, filesize=5.3 K 2024-11-23T05:55:03,949 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false 2024-11-23T05:55:03,949 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T05:55:03,950 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/b5ad71ed15a6424e83c0c2b775a703a9 2024-11-23T05:55:03,951 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/c5effd87a2e64c6e913c88954bee422c 2024-11-23T05:55:03,952 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/2c07fde6fdb84e719be04e043dc76fee 2024-11-23T05:55:03,956 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-23T05:55:03,956 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T05:55:03,957 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T05:55:03,957 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732341303858Running coprocessor pre-close hooks at 1732341303858Disabling compacts and flushes for region at 1732341303858Disabling writes for close at 1732341303858Obtaining lock to block concurrent updates at 1732341303858Preparing flush snapshotting stores in 1588230740 at 1732341303858Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1732341303859 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732341303859Flushing 1588230740/info: creating writer at 1732341303859Flushing 1588230740/info: appending metadata at 1732341303885 (+26 ms)Flushing 1588230740/info: closing flushed file at 1732341303885Flushing 1588230740/ns: creating writer at 1732341303891 (+6 ms)Flushing 1588230740/ns: appending metadata at 1732341303910 (+19 ms)Flushing 1588230740/ns: closing flushed file at 1732341303910Flushing 1588230740/table: creating writer at 1732341303914 (+4 ms)Flushing 1588230740/table: appending metadata at 1732341303936 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732341303936Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e24af2d: reopening flushed file at 1732341303939 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc23a3c: reopening flushed file at 1732341303943 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b14cb5e: reopening flushed file at 1732341303945 (+2 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 90ms, sequenceid=15, compaction requested=false at 1732341303949 (+4 ms)Writing region close event to WAL at 1732341303952 (+3 ms)Running coprocessor post-close hooks at 1732341303956 (+4 ms)Closed at 1732341303957 (+1 ms) 2024-11-23T05:55:03,957 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T05:55:03,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:03,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46405-0x101678c04fe0001, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:03,987 INFO [RS:0;9abfc7c2569e:46405 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:55:03,987 INFO [RS:0;9abfc7c2569e:46405 {}] regionserver.HRegionServer(1031): Exiting; stopping=9abfc7c2569e,46405,1732341299042; zookeeper connection closed. 2024-11-23T05:55:03,987 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@de38f15 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@de38f15 2024-11-23T05:55:04,058 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(976): stopping server 9abfc7c2569e,37149,1732341299177; all regions closed. 2024-11-23T05:55:04,063 DEBUG [RS:1;9abfc7c2569e:37149 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs 2024-11-23T05:55:04,063 INFO [RS:1;9abfc7c2569e:37149 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C37149%2C1732341299177.meta:.meta(num 1732341301101) 2024-11-23T05:55:04,067 DEBUG [RS:1;9abfc7c2569e:37149 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs 2024-11-23T05:55:04,067 INFO [RS:1;9abfc7c2569e:37149 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C37149%2C1732341299177:(num 1732341300691) 2024-11-23T05:55:04,067 DEBUG [RS:1;9abfc7c2569e:37149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:04,067 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:04,067 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:55:04,067 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.ChoreService(370): Chore service for: regionserver/9abfc7c2569e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T05:55:04,068 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:55:04,068 INFO [regionserver/9abfc7c2569e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:55:04,068 INFO [RS:1;9abfc7c2569e:37149 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37149 2024-11-23T05:55:04,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:04,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9abfc7c2569e,37149,1732341299177 2024-11-23T05:55:04,084 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:55:04,085 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f51b4477b20@e565a48 rejected from java.util.concurrent.ThreadPoolExecutor@4b54c5e6[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-23T05:55:04,094 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9abfc7c2569e,37149,1732341299177] 2024-11-23T05:55:04,102 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9abfc7c2569e,37149,1732341299177 already deleted, retry=false 2024-11-23T05:55:04,102 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9abfc7c2569e,37149,1732341299177 expired; onlineServers=0 2024-11-23T05:55:04,102 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9abfc7c2569e,46451,1732341298038' ***** 2024-11-23T05:55:04,102 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T05:55:04,103 INFO [M:0;9abfc7c2569e:46451 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:55:04,103 INFO [M:0;9abfc7c2569e:46451 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:55:04,103 DEBUG [M:0;9abfc7c2569e:46451 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T05:55:04,103 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T05:55:04,103 DEBUG [M:0;9abfc7c2569e:46451 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T05:55:04,103 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732341300329 {}] cleaner.HFileCleaner(306): Exit Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732341300329,5,FailOnTimeoutGroup] 2024-11-23T05:55:04,103 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732341300330 {}] cleaner.HFileCleaner(306): Exit Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732341300330,5,FailOnTimeoutGroup] 2024-11-23T05:55:04,103 INFO [M:0;9abfc7c2569e:46451 {}] hbase.ChoreService(370): Chore service for: master/9abfc7c2569e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T05:55:04,104 INFO [M:0;9abfc7c2569e:46451 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:55:04,104 DEBUG [M:0;9abfc7c2569e:46451 {}] master.HMaster(1795): Stopping service threads 2024-11-23T05:55:04,104 INFO [M:0;9abfc7c2569e:46451 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T05:55:04,104 INFO [M:0;9abfc7c2569e:46451 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T05:55:04,104 INFO [M:0;9abfc7c2569e:46451 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T05:55:04,105 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T05:55:04,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T05:55:04,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,111 DEBUG [M:0;9abfc7c2569e:46451 {}] zookeeper.ZKUtil(347): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T05:55:04,111 WARN [M:0;9abfc7c2569e:46451 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T05:55:04,111 INFO [M:0;9abfc7c2569e:46451 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.lastflushedseqids 2024-11-23T05:55:04,122 INFO [M:0;9abfc7c2569e:46451 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T05:55:04,122 INFO [M:0;9abfc7c2569e:46451 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T05:55:04,123 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T05:55:04,123 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,123 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,123 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T05:55:04,123 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,123 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.66 KB heapSize=42.99 KB 2024-11-23T05:55:04,132 DEBUG [M:0;9abfc7c2569e:46451 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 92e3f23e53994ceba634545e7d5a60fa 2024-11-23T05:55:04,145 DEBUG [M:0;9abfc7c2569e:46451 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/92e3f23e53994ceba634545e7d5a60fa is 82, key is hbase:meta,,1/info:regioninfo/1732341301159/Put/seqid=0 2024-11-23T05:55:04,146 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/92e3f23e53994ceba634545e7d5a60fa 2024-11-23T05:55:04,153 DEBUG [M:0;9abfc7c2569e:46451 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 13997d71d666482cbfb266da3ca9c4a6 2024-11-23T05:55:04,170 DEBUG [M:0;9abfc7c2569e:46451 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13997d71d666482cbfb266da3ca9c4a6 is 777, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732341301875/Put/seqid=0 2024-11-23T05:55:04,171 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.04 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13997d71d666482cbfb266da3ca9c4a6 2024-11-23T05:55:04,179 DEBUG [M:0;9abfc7c2569e:46451 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8ff6b0ec1ec140159e78fa4444848662 2024-11-23T05:55:04,192 DEBUG [M:0;9abfc7c2569e:46451 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8ff6b0ec1ec140159e78fa4444848662 is 69, key is 9abfc7c2569e,37149,1732341299177/rs:state/1732341300407/Put/seqid=0 2024-11-23T05:55:04,192 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8ff6b0ec1ec140159e78fa4444848662 2024-11-23T05:55:04,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:04,194 INFO [RS:1;9abfc7c2569e:37149 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:55:04,194 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37149-0x101678c04fe0002, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:04,194 INFO [RS:1;9abfc7c2569e:37149 {}] regionserver.HRegionServer(1031): Exiting; stopping=9abfc7c2569e,37149,1732341299177; zookeeper connection closed. 2024-11-23T05:55:04,194 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/92e3f23e53994ceba634545e7d5a60fa as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/92e3f23e53994ceba634545e7d5a60fa 2024-11-23T05:55:04,194 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4c85063d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4c85063d 2024-11-23T05:55:04,195 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-23T05:55:04,196 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/92e3f23e53994ceba634545e7d5a60fa, entries=8, sequenceid=84, filesize=5.5 K 2024-11-23T05:55:04,196 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/13997d71d666482cbfb266da3ca9c4a6 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/13997d71d666482cbfb266da3ca9c4a6 2024-11-23T05:55:04,198 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/13997d71d666482cbfb266da3ca9c4a6, entries=9, sequenceid=84, filesize=6.5 K 2024-11-23T05:55:04,198 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8ff6b0ec1ec140159e78fa4444848662 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8ff6b0ec1ec140159e78fa4444848662 2024-11-23T05:55:04,200 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8ff6b0ec1ec140159e78fa4444848662, entries=2, sequenceid=84, filesize=5.1 K 2024-11-23T05:55:04,201 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false 2024-11-23T05:55:04,203 INFO [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,203 DEBUG [M:0;9abfc7c2569e:46451 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732341304123Disabling compacts and flushes for region at 1732341304123Disabling writes for close at 1732341304123Obtaining lock to block concurrent updates at 1732341304123Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732341304123Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35494, getHeapSize=43960, getOffHeapSize=0, getCellsCount=99 at 1732341304124 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732341304124Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732341304124Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732341304145 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732341304145Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732341304148 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732341304170 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732341304170Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732341304174 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732341304191 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732341304191Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ffd2fed: reopening flushed file at 1732341304194 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e0cbafd: reopening flushed file at 1732341304196 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65d35a8: reopening flushed file at 1732341304198 (+2 ms)Finished flush of dataSize ~34.66 KB/35494, heapSize ~42.70 KB/43720, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 78ms, sequenceid=84, compaction requested=false at 1732341304201 (+3 ms)Writing region close event to WAL at 1732341304202 (+1 ms)Closed at 1732341304203 (+1 ms) 2024-11-23T05:55:04,204 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:55:04,205 INFO [M:0;9abfc7c2569e:46451 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T05:55:04,205 INFO [M:0;9abfc7c2569e:46451 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46451 2024-11-23T05:55:04,205 INFO [M:0;9abfc7c2569e:46451 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:55:04,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:04,336 INFO [M:0;9abfc7c2569e:46451 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:55:04,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46451-0x101678c04fe0000, quorum=127.0.0.1:64102, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:04,343 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T05:55:04,366 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T05:55:04,367 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,374 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=51 (was 12) Potentially hanging thread: master/9abfc7c2569e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/9abfc7c2569e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1732341302219 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732341303941 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: regionserver/9abfc7c2569e:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/9abfc7c2569e:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732341303944 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1732341303764 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=354 (was 365), ProcessCount=11 (was 11), AvailableMemoryMB=7723 (was 8134) 2024-11-23T05:55:04,380 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=51, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=354, ProcessCount=11, AvailableMemoryMB=7723 2024-11-23T05:55:04,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/hadoop-log-dir so I do NOT create it in target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d 2024-11-23T05:55:04,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/hadoop-tmp-dir so I do NOT create it in target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d 2024-11-23T05:55:04,380 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,382 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/cluster_5e91da26-5500-2c34-32e8-19f0085779b4, deleteOnExit=true 2024-11-23T05:55:04,385 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/cluster_5e91da26-5500-2c34-32e8-19f0085779b4/zookeeper_0, clientPort=51849, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/cluster_5e91da26-5500-2c34-32e8-19f0085779b4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/cluster_5e91da26-5500-2c34-32e8-19f0085779b4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T05:55:04,386 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51849 2024-11-23T05:55:04,397 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a with version=8 2024-11-23T05:55:04,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/hbase-staging 2024-11-23T05:55:04,400 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:55:04,400 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,401 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:55:04,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:55:04,401 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T05:55:04,401 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:55:04,402 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36109 2024-11-23T05:55:04,404 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36109 connecting to ZooKeeper ensemble=127.0.0.1:51849 2024-11-23T05:55:04,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361090x0, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:55:04,441 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36109-0x101678c1f050000 connected 2024-11-23T05:55:04,514 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,515 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,516 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:04,516 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a, hbase.cluster.distributed=false 2024-11-23T05:55:04,518 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:55:04,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36109 2024-11-23T05:55:04,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36109 2024-11-23T05:55:04,519 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36109 2024-11-23T05:55:04,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36109 2024-11-23T05:55:04,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36109 2024-11-23T05:55:04,538 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:55:04,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,538 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:55:04,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:55:04,539 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T05:55:04,539 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:55:04,539 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42619 2024-11-23T05:55:04,541 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42619 connecting to ZooKeeper ensemble=127.0.0.1:51849 2024-11-23T05:55:04,541 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,541 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,548 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:426190x0, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:55:04,548 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:426190x0, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:04,548 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42619-0x101678c1f050001 connected 2024-11-23T05:55:04,549 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T05:55:04,549 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.cache 2024-11-23T05:55:04,549 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T05:55:04,550 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T05:55:04,550 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T05:55:04,551 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T05:55:04,551 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T05:55:04,551 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T05:55:04,551 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence is complete. 2024-11-23T05:55:04,551 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T05:55:04,555 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T05:55:04,556 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T05:55:04,557 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:55:04,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42619 2024-11-23T05:55:04,558 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42619 2024-11-23T05:55:04,560 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42619 2024-11-23T05:55:04,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42619 2024-11-23T05:55:04,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42619 2024-11-23T05:55:04,577 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:55:04,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,577 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:55:04,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:04,577 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:55:04,577 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T05:55:04,577 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:55:04,578 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44239 2024-11-23T05:55:04,580 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44239 connecting to ZooKeeper ensemble=127.0.0.1:51849 2024-11-23T05:55:04,580 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,580 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:442390x0, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:55:04,590 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:442390x0, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:04,590 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44239-0x101678c1f050002 connected 2024-11-23T05:55:04,590 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T05:55:04,591 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.cache 2024-11-23T05:55:04,591 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T05:55:04,591 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T05:55:04,591 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T05:55:04,592 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T05:55:04,592 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-23T05:55:04,592 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T05:55:04,592 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T05:55:04,592 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T05:55:04,592 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence is complete. 2024-11-23T05:55:04,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T05:55:04,594 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:55:04,596 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44239 2024-11-23T05:55:04,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44239 2024-11-23T05:55:04,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44239 2024-11-23T05:55:04,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44239 2024-11-23T05:55:04,599 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44239 2024-11-23T05:55:04,610 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9abfc7c2569e:36109 2024-11-23T05:55:04,611 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:04,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:55:04,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:55:04,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:55:04,622 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:04,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T05:55:04,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T05:55:04,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,632 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T05:55:04,632 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9abfc7c2569e,36109,1732341304400 from backup master directory 2024-11-23T05:55:04,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:55:04,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:04,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:55:04,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:55:04,639 WARN [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:55:04,640 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:04,641 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/hbase.id] with ID: 3e36e304-43b7-4c14-b16f-bb4dc11783de 2024-11-23T05:55:04,641 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.tmp/hbase.id 2024-11-23T05:55:04,645 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/hbase.id] 2024-11-23T05:55:04,674 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:04,675 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T05:55:04,679 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 3ms. 2024-11-23T05:55:04,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,695 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:55:04,696 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T05:55:04,697 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:04,716 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store 2024-11-23T05:55:04,721 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:04,721 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T05:55:04,721 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,721 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,721 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T05:55:04,721 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,722 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:04,722 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732341304721Disabling compacts and flushes for region at 1732341304721Disabling writes for close at 1732341304721Writing region close event to WAL at 1732341304721Closed at 1732341304722 (+1 ms) 2024-11-23T05:55:04,722 WARN [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/.initializing 2024-11-23T05:55:04,722 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/WALs/9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:04,727 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:55:04,730 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9abfc7c2569e%2C36109%2C1732341304400, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/WALs/9abfc7c2569e,36109,1732341304400, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/oldWALs, maxLogs=10 2024-11-23T05:55:04,745 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/WALs/9abfc7c2569e,36109,1732341304400/9abfc7c2569e%2C36109%2C1732341304400.1732341304730 2024-11-23T05:55:04,746 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:04,746 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:04,746 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:04,746 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,746 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T05:55:04,751 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:04,752 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T05:55:04,755 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:04,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T05:55:04,758 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:04,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,761 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T05:55:04,761 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,762 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:04,762 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,762 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,763 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,763 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,763 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,763 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T05:55:04,763 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:55:04,768 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:55:04,768 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73671393, jitterRate=0.09778930246829987}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T05:55:04,769 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732341304746Initializing all the Stores at 1732341304747 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341304747Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341304747Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341304747Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341304747Cleaning up temporary data from old regions at 1732341304763 (+16 ms)Region opened successfully at 1732341304769 (+6 ms) 2024-11-23T05:55:04,769 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T05:55:04,772 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf577b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:55:04,773 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T05:55:04,774 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T05:55:04,774 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T05:55:04,774 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T05:55:04,774 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-23T05:55:04,774 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-23T05:55:04,774 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T05:55:04,776 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T05:55:04,777 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T05:55:04,806 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T05:55:04,806 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T05:55:04,807 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T05:55:04,814 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T05:55:04,815 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T05:55:04,816 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T05:55:04,823 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T05:55:04,824 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T05:55:04,831 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T05:55:04,833 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T05:55:04,839 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T05:55:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,849 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9abfc7c2569e,36109,1732341304400, sessionid=0x101678c1f050000, setting cluster-up flag (Was=false) 2024-11-23T05:55:04,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,889 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T05:55:04,891 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:04,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,906 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:04,931 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T05:55:04,932 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:04,933 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T05:55:04,934 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:04,934 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T05:55:04,935 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T05:55:04,935 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9abfc7c2569e,36109,1732341304400 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9abfc7c2569e:0, corePoolSize=10, maxPoolSize=10 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:55:04,937 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:04,939 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732341334939 2024-11-23T05:55:04,939 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:04,939 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T05:55:04,939 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T05:55:04,939 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T05:55:04,939 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T05:55:04,939 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T05:55:04,939 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T05:55:04,939 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T05:55:04,940 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,940 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:04,940 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T05:55:04,940 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T05:55:04,940 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T05:55:04,940 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T05:55:04,941 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T05:55:04,941 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T05:55:04,942 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732341304941,5,FailOnTimeoutGroup] 2024-11-23T05:55:04,946 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732341304942,5,FailOnTimeoutGroup] 2024-11-23T05:55:04,947 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:04,947 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T05:55:04,947 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:04,947 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:04,959 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T05:55:04,959 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:04,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:04,968 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T05:55:04,971 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T05:55:04,972 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:04,972 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T05:55:04,975 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T05:55:04,975 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:04,975 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T05:55:04,978 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T05:55:04,978 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:04,978 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T05:55:04,981 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T05:55:04,981 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:04,981 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:04,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T05:55:04,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:04,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:04,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T05:55:04,982 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T05:55:04,983 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T05:55:04,983 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T05:55:04,989 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:55:04,989 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61135553, jitterRate=-0.08900927007198334}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T05:55:04,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732341304967Initializing all the Stores at 1732341304967Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341304968 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341304968Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341304968Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341304968Cleaning up temporary data from old regions at 1732341304982 (+14 ms)Region opened successfully at 1732341304991 (+9 ms) 2024-11-23T05:55:04,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T05:55:04,991 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T05:55:04,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T05:55:04,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T05:55:04,991 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T05:55:04,992 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T05:55:04,992 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732341304991Disabling compacts and flushes for region at 1732341304991Disabling writes for close at 1732341304991Writing region close event to WAL at 1732341304992 (+1 ms)Closed at 1732341304992 2024-11-23T05:55:04,993 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:04,993 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T05:55:04,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T05:55:04,995 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T05:55:04,996 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T05:55:05,001 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(746): ClusterId : 3e36e304-43b7-4c14-b16f-bb4dc11783de 2024-11-23T05:55:05,001 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(746): ClusterId : 3e36e304-43b7-4c14-b16f-bb4dc11783de 2024-11-23T05:55:05,001 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T05:55:05,001 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T05:55:05,024 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T05:55:05,024 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T05:55:05,024 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T05:55:05,024 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T05:55:05,032 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T05:55:05,032 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T05:55:05,032 DEBUG [RS:0;9abfc7c2569e:42619 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c691d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:55:05,033 DEBUG [RS:1;9abfc7c2569e:44239 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14da34f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:55:05,043 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9abfc7c2569e:42619 2024-11-23T05:55:05,043 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;9abfc7c2569e:44239 2024-11-23T05:55:05,043 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T05:55:05,044 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T05:55:05,044 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T05:55:05,044 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T05:55:05,044 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T05:55:05,044 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T05:55:05,045 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(2659): reportForDuty to master=9abfc7c2569e,36109,1732341304400 with port=42619, startcode=1732341304537 2024-11-23T05:55:05,045 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(2659): reportForDuty to master=9abfc7c2569e,36109,1732341304400 with port=44239, startcode=1732341304577 2024-11-23T05:55:05,045 DEBUG [RS:1;9abfc7c2569e:44239 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T05:55:05,045 DEBUG [RS:0;9abfc7c2569e:42619 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T05:55:05,048 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47185, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T05:55:05,048 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47977, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T05:55:05,049 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:05,049 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36109 {}] master.ServerManager(517): Registering regionserver=9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:05,050 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,050 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36109 {}] master.ServerManager(517): Registering regionserver=9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,051 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:05,051 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T05:55:05,051 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T05:55:05,052 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:05,052 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T05:55:05,052 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T05:55:05,056 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:05,088 DEBUG [RS:0;9abfc7c2569e:42619 {}] zookeeper.ZKUtil(111): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:05,088 WARN [RS:0;9abfc7c2569e:42619 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:55:05,088 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9abfc7c2569e,42619,1732341304537] 2024-11-23T05:55:05,089 INFO [RS:0;9abfc7c2569e:42619 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:05,089 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9abfc7c2569e,44239,1732341304577] 2024-11-23T05:55:05,089 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:05,089 DEBUG [RS:1;9abfc7c2569e:44239 {}] zookeeper.ZKUtil(111): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,089 WARN [RS:1;9abfc7c2569e:44239 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:55:05,089 INFO [RS:1;9abfc7c2569e:44239 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:05,089 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,093 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T05:55:05,093 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T05:55:05,095 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T05:55:05,096 INFO [RS:1;9abfc7c2569e:44239 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T05:55:05,096 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,096 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T05:55:05,097 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T05:55:05,097 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,097 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,097 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,097 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,097 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,097 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,097 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:55:05,097 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,098 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,098 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,098 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,098 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,098 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,098 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:05,098 DEBUG [RS:1;9abfc7c2569e:44239 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:05,098 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T05:55:05,103 INFO [RS:0;9abfc7c2569e:42619 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T05:55:05,103 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,103 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,103 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,103 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,103 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,103 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,103 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,44239,1732341304577-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:55:05,104 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T05:55:05,104 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T05:55:05,104 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,104 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,104 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,105 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:05,106 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:05,106 DEBUG [RS:0;9abfc7c2569e:42619 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:05,106 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,106 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,107 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,107 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,107 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,107 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,42619,1732341304537-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:55:05,122 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T05:55:05,122 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T05:55:05,122 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,42619,1732341304537-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,122 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,44239,1732341304577-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,122 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,122 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,123 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.Replication(171): 9abfc7c2569e,44239,1732341304577 started 2024-11-23T05:55:05,123 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.Replication(171): 9abfc7c2569e,42619,1732341304537 started 2024-11-23T05:55:05,138 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,138 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,138 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1482): Serving as 9abfc7c2569e,44239,1732341304577, RpcServer on 9abfc7c2569e/172.17.0.2:44239, sessionid=0x101678c1f050002 2024-11-23T05:55:05,138 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1482): Serving as 9abfc7c2569e,42619,1732341304537, RpcServer on 9abfc7c2569e/172.17.0.2:42619, sessionid=0x101678c1f050001 2024-11-23T05:55:05,138 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T05:55:05,138 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T05:55:05,138 DEBUG [RS:1;9abfc7c2569e:44239 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,138 DEBUG [RS:0;9abfc7c2569e:42619 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:05,138 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,44239,1732341304577' 2024-11-23T05:55:05,138 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,42619,1732341304537' 2024-11-23T05:55:05,138 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T05:55:05,138 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T05:55:05,139 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T05:55:05,139 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T05:55:05,139 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T05:55:05,139 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T05:55:05,139 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T05:55:05,139 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T05:55:05,139 DEBUG [RS:1;9abfc7c2569e:44239 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,139 DEBUG [RS:0;9abfc7c2569e:42619 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:05,139 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,42619,1732341304537' 2024-11-23T05:55:05,139 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,44239,1732341304577' 2024-11-23T05:55:05,139 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T05:55:05,139 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T05:55:05,140 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T05:55:05,140 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T05:55:05,140 DEBUG [RS:1;9abfc7c2569e:44239 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T05:55:05,140 DEBUG [RS:0;9abfc7c2569e:42619 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T05:55:05,140 INFO [RS:1;9abfc7c2569e:44239 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T05:55:05,140 INFO [RS:0;9abfc7c2569e:42619 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T05:55:05,140 INFO [RS:1;9abfc7c2569e:44239 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T05:55:05,140 INFO [RS:0;9abfc7c2569e:42619 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T05:55:05,146 WARN [9abfc7c2569e:36109 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-23T05:55:05,241 INFO [RS:0;9abfc7c2569e:42619 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:55:05,241 INFO [RS:1;9abfc7c2569e:44239 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:55:05,242 INFO [RS:1;9abfc7c2569e:44239 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9abfc7c2569e%2C44239%2C1732341304577, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,44239,1732341304577, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs, maxLogs=55 2024-11-23T05:55:05,242 INFO [RS:0;9abfc7c2569e:42619 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9abfc7c2569e%2C42619%2C1732341304537, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,42619,1732341304537, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs, maxLogs=55 2024-11-23T05:55:05,260 INFO [RS:0;9abfc7c2569e:42619 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,42619,1732341304537/9abfc7c2569e%2C42619%2C1732341304537.1732341305244 2024-11-23T05:55:05,260 INFO [RS:1;9abfc7c2569e:44239 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,44239,1732341304577/9abfc7c2569e%2C44239%2C1732341304577.1732341305244 2024-11-23T05:55:05,260 DEBUG [RS:0;9abfc7c2569e:42619 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:05,260 DEBUG [RS:1;9abfc7c2569e:44239 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:05,396 DEBUG [9abfc7c2569e:36109 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-23T05:55:05,397 DEBUG [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(204): Hosts are {9abfc7c2569e=0} racks are {/default-rack=0} 2024-11-23T05:55:05,398 DEBUG [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T05:55:05,398 DEBUG [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T05:55:05,398 DEBUG [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T05:55:05,398 DEBUG [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T05:55:05,398 INFO [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T05:55:05,398 INFO [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T05:55:05,398 DEBUG [9abfc7c2569e:36109 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T05:55:05,399 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,400 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9abfc7c2569e,44239,1732341304577, state=OPENING 2024-11-23T05:55:05,431 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T05:55:05,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:05,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:05,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:05,442 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T05:55:05,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:05,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9abfc7c2569e,44239,1732341304577}] 2024-11-23T05:55:05,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:05,442 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:05,598 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T05:55:05,603 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38463, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T05:55:05,607 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T05:55:05,607 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:05,608 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T05:55:05,610 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9abfc7c2569e%2C44239%2C1732341304577.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,44239,1732341304577, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs, maxLogs=55 2024-11-23T05:55:05,626 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,44239,1732341304577/9abfc7c2569e%2C44239%2C1732341304577.meta.1732341305611.meta 2024-11-23T05:55:05,626 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:05,627 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:05,627 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T05:55:05,627 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T05:55:05,627 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T05:55:05,628 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T05:55:05,628 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:05,628 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T05:55:05,628 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T05:55:05,630 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-23T05:55:05,631 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T05:55:05,631 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:05,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:05,631 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-23T05:55:05,632 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T05:55:05,632 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:05,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:05,632 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-23T05:55:05,633 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T05:55:05,633 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:05,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:05,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-23T05:55:05,633 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T05:55:05,633 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:05,633 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:55:05,634 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T05:55:05,634 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:05,635 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740 2024-11-23T05:55:05,635 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T05:55:05,635 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T05:55:05,635 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T05:55:05,636 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T05:55:05,636 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75006208, jitterRate=0.11767959594726562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T05:55:05,636 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T05:55:05,637 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732341305628Writing region info on filesystem at 1732341305628Initializing all the Stores at 1732341305628Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341305628Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341305630 (+2 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341305630Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732341305630Cleaning up temporary data from old regions at 1732341305635 (+5 ms)Running coprocessor post-open hooks at 1732341305636 (+1 ms)Region opened successfully at 1732341305637 (+1 ms) 2024-11-23T05:55:05,637 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732341305598 2024-11-23T05:55:05,640 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T05:55:05,640 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T05:55:05,641 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,641 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9abfc7c2569e,44239,1732341304577, state=OPEN 2024-11-23T05:55:05,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:55:05,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:55:05,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:55:05,673 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:05,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:05,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:05,673 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:55:05,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T05:55:05,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9abfc7c2569e,44239,1732341304577 in 231 msec 2024-11-23T05:55:05,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T05:55:05,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 681 msec 2024-11-23T05:55:05,678 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:55:05,678 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T05:55:05,681 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:55:05,681 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,44239,1732341304577, seqNum=-1] 2024-11-23T05:55:05,682 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:55:05,684 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45995, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:55:05,689 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 755 msec 2024-11-23T05:55:05,689 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732341305689, completionTime=-1 2024-11-23T05:55:05,689 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-23T05:55:05,689 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T05:55:05,692 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-23T05:55:05,692 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732341365692 2024-11-23T05:55:05,692 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732341425692 2024-11-23T05:55:05,692 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-23T05:55:05,692 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-23T05:55:05,693 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36109,1732341304400-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,693 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36109,1732341304400-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,693 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36109,1732341304400-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,693 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9abfc7c2569e:36109, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,693 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,694 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,696 DEBUG [master/9abfc7c2569e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T05:55:05,698 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.058sec 2024-11-23T05:55:05,698 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T05:55:05,698 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T05:55:05,699 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T05:55:05,699 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T05:55:05,699 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T05:55:05,699 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36109,1732341304400-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:55:05,699 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36109,1732341304400-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T05:55:05,699 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T05:55:05,700 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T05:55:05,700 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36109,1732341304400-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:05,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dccad78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:55:05,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,36109,-1 for getting cluster id 2024-11-23T05:55:05,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:55:05,703 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '3e36e304-43b7-4c14-b16f-bb4dc11783de' 2024-11-23T05:55:05,704 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:55:05,704 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "3e36e304-43b7-4c14-b16f-bb4dc11783de" 2024-11-23T05:55:05,704 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@856a9c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:55:05,704 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,36109,-1] 2024-11-23T05:55:05,704 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:55:05,705 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:05,706 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:55:05,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d4294a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:55:05,708 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:55:05,710 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,44239,1732341304577, seqNum=-1] 2024-11-23T05:55:05,710 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:55:05,712 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:55:05,714 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:05,714 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:55:05,716 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:05,716 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d95000 2024-11-23T05:55:05,716 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:55:05,718 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:55:05,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36109 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1732341305714', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:55:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36109 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 2024-11-23T05:55:05,722 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:55:05,722 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:05,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36109 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1732341305714" procId is: 4 2024-11-23T05:55:05,723 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:55:05,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:05,737 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aa214f0922457f1679e5487887d642e3, NAME => 'testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1732341305714', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:05,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:05,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(1722): Closing aa214f0922457f1679e5487887d642e3, disabling compactions & flushes 2024-11-23T05:55:05,743 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:05,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:05,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. after waiting 0 ms 2024-11-23T05:55:05,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:05,743 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:05,743 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1732341305714-pool-0 {}] regionserver.HRegion(1676): Region close journal for aa214f0922457f1679e5487887d642e3: Waiting for close lock at 1732341305743Disabling compacts and flushes for region at 1732341305743Disabling writes for close at 1732341305743Writing region close event to WAL at 1732341305743Closed at 1732341305743 2024-11-23T05:55:05,744 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:55:05,745 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1732341305744"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732341305744"}]},"ts":"1732341305744"} 2024-11-23T05:55:05,747 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:55:05,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:55:05,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732341305714","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732341305748"}]},"ts":"1732341305748"} 2024-11-23T05:55:05,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732341305714, state=ENABLING in hbase:meta 2024-11-23T05:55:05,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {9abfc7c2569e=0} racks are {/default-rack=0} 2024-11-23T05:55:05,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-23T05:55:05,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-23T05:55:05,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-23T05:55:05,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-23T05:55:05,751 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-23T05:55:05,751 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-23T05:55:05,751 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-23T05:55:05,752 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN}] 2024-11-23T05:55:05,753 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN 2024-11-23T05:55:05,754 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN; state=OFFLINE, location=9abfc7c2569e,42619,1732341304537; forceNewPlan=false, retain=false 2024-11-23T05:55:05,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:05,904 INFO [9abfc7c2569e:36109 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-23T05:55:05,905 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aa214f0922457f1679e5487887d642e3, regionState=OPENING, regionLocation=9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:05,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN because future has completed 2024-11-23T05:55:05,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa214f0922457f1679e5487887d642e3, server=9abfc7c2569e,42619,1732341304537}] 2024-11-23T05:55:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:06,061 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T05:55:06,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58869, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T05:55:06,069 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:06,069 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => aa214f0922457f1679e5487887d642e3, NAME => 'testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:06,070 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732341305714 aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,070 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:06,070 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,070 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,072 INFO [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,075 INFO [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa214f0922457f1679e5487887d642e3 columnFamilyName family 2024-11-23T05:55:06,076 DEBUG [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:06,076 INFO [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] regionserver.HStore(327): Store=aa214f0922457f1679e5487887d642e3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:06,076 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,077 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,077 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,077 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,077 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,078 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,083 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:55:06,084 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened aa214f0922457f1679e5487887d642e3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59846206, jitterRate=-0.10822203755378723}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T05:55:06,084 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:06,085 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for aa214f0922457f1679e5487887d642e3: Running coprocessor pre-open hook at 1732341306071Writing region info on filesystem at 1732341306071Initializing all the Stores at 1732341306071Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341306071Cleaning up temporary data from old regions at 1732341306077 (+6 ms)Running coprocessor post-open hooks at 1732341306084 (+7 ms)Region opened successfully at 1732341306085 (+1 ms) 2024-11-23T05:55:06,086 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3., pid=6, masterSystemTime=1732341306060 2024-11-23T05:55:06,089 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=aa214f0922457f1679e5487887d642e3, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:06,089 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:06,089 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:06,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa214f0922457f1679e5487887d642e3, server=9abfc7c2569e,42619,1732341304537 because future has completed 2024-11-23T05:55:06,094 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T05:55:06,095 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure aa214f0922457f1679e5487887d642e3, server=9abfc7c2569e,42619,1732341304537 in 184 msec 2024-11-23T05:55:06,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T05:55:06,096 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN in 342 msec 2024-11-23T05:55:06,097 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:55:06,097 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1732341305714","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732341306097"}]},"ts":"1732341306097"} 2024-11-23T05:55:06,099 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1732341305714, state=ENABLED in hbase:meta 2024-11-23T05:55:06,100 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:55:06,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1732341305714 in 380 msec 2024-11-23T05:55:06,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36109 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:55:06,348 INFO [RPCClient-NioEventLoopGroup-5-9 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1732341305714 completed 2024-11-23T05:55:06,348 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1732341305714 get assigned. Timeout = 60000ms 2024-11-23T05:55:06,348 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T05:55:06,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1732341305714 assigned to meta. Checking AM states. 2024-11-23T05:55:06,353 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-23T05:55:06,353 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1732341305714 assigned. 2024-11-23T05:55:06,359 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1732341305714', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3., hostname=9abfc7c2569e,42619,1732341304537, seqNum=2] 2024-11-23T05:55:06,359 DEBUG [RPCClient-NioEventLoopGroup-5-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:55:06,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:55:06,365 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing aa214f0922457f1679e5487887d642e3 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-23T05:55:06,374 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fe992f6d17dc488caf32bd3e74606848 2024-11-23T05:55:06,387 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/.tmp/family/fe992f6d17dc488caf32bd3e74606848 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-23T05:55:06,388 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/.tmp/family/fe992f6d17dc488caf32bd3e74606848 2024-11-23T05:55:06,391 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/.tmp/family/fe992f6d17dc488caf32bd3e74606848 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848 2024-11-23T05:55:06,392 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848, delay=999 ms 2024-11-23T05:55:06,393 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848, entries=2, sequenceid=6, filesize=4.9 K 2024-11-23T05:55:06,394 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for aa214f0922457f1679e5487887d642e3 in 29ms, sequenceid=6, compaction requested=false 2024-11-23T05:55:06,394 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1732341305714' 2024-11-23T05:55:06,394 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for aa214f0922457f1679e5487887d642e3: 2024-11-23T05:55:06,633 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:55:07,392 DEBUG [hfile-prefetch-1732341303764 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fe992f6d17dc488caf32bd3e74606848_0. 2024-11-23T05:55:07,393 DEBUG [hfile-prefetch-1732341303764 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key fe992f6d17dc488caf32bd3e74606848_117. 2024-11-23T05:55:07,394 DEBUG [hfile-prefetch-1732341303764 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848, with total blocks 2, and data blocks 1 2024-11-23T05:55:07,394 DEBUG [hfile-prefetch-1732341303764 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-11-23T05:55:07,394 DEBUG [hfile-prefetch-1732341303764 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-23T05:55:07,394 DEBUG [hfile-prefetch-1732341303764 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848. Will sleep 100ms and try the verification again. 2024-11-23T05:55:07,395 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:0;9abfc7c2569e:42619,5,FailOnTimeoutGroup] 2024-11-23T05:55:07,395 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9abfc7c2569e,42619,1732341304537' ***** 2024-11-23T05:55:07,395 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 0 2024-11-23T05:55:07,396 INFO [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T05:55:07,409 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T05:55:07,410 INFO [regionserver/9abfc7c2569e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:07,472 INFO [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence 2024-11-23T05:55:07,472 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T05:55:07,472 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T05:55:07,472 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T05:55:07,472 INFO [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T05:55:07,472 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T05:55:07,482 DEBUG [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1487): Thread RS:0;9abfc7c2569e:42619 started persisting bucket cache to file 2024-11-23T05:55:07,482 DEBUG [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:07,482 DEBUG [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:07,487 DEBUG [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-23T05:55:07,487 DEBUG [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1732341307482 2024-11-23T05:55:07,487 DEBUG [RS:0;9abfc7c2569e:42619 {}] bucket.BucketCache(1509): Thread RS:0;9abfc7c2569e:42619 finished persisting bucket cache to file, renaming 2024-11-23T05:55:07,487 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T05:55:07,488 INFO [RS:0;9abfc7c2569e:42619 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T05:55:07,488 INFO [RS:0;9abfc7c2569e:42619 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T05:55:07,488 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(3091): Received CLOSE for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:07,488 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(959): stopping server 9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:07,489 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:55:07,489 INFO [RS:0;9abfc7c2569e:42619 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9abfc7c2569e:42619. 2024-11-23T05:55:07,489 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T05:55:07,489 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa214f0922457f1679e5487887d642e3, disabling compactions & flushes 2024-11-23T05:55:07,489 DEBUG [RS:0;9abfc7c2569e:42619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:07,489 DEBUG [RS:0;9abfc7c2569e:42619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:07,489 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:07,489 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:07,489 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T05:55:07,489 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. after waiting 0 ms 2024-11-23T05:55:07,489 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1325): Online Regions={aa214f0922457f1679e5487887d642e3=testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.} 2024-11-23T05:55:07,489 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:07,490 DEBUG [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1351): Waiting on aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:07,491 DEBUG [StoreFileCloser-aa214f0922457f1679e5487887d642e3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848 2024-11-23T05:55:07,491 WARN [hfile-prefetch-1732341303764 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-11-23T05:55:07,491 DEBUG [hfile-prefetch-1732341303764 {}] hfile.PrefetchExecutor(121): Prefetch completed for fe992f6d17dc488caf32bd3e74606848 2024-11-23T05:55:07,494 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-23T05:55:07,494 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-23T05:55:07,494 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:07,494 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-23T05:55:07,494 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:07,495 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:07,495 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa214f0922457f1679e5487887d642e3: Waiting for close lock at 1732341307489Running coprocessor pre-close hooks at 1732341307489Disabling compacts and flushes for region at 1732341307489Disabling writes for close at 1732341307489Writing region close event to WAL at 1732341307491 (+2 ms)Running coprocessor post-close hooks at 1732341307495 (+4 ms)Closed at 1732341307495 2024-11-23T05:55:07,495 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:07,496 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T05:55:07,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732341307494 2024-11-23T05:55:07,497 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-23T05:55:07,690 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(976): stopping server 9abfc7c2569e,42619,1732341304537; all regions closed. 2024-11-23T05:55:07,692 DEBUG [RS:0;9abfc7c2569e:42619 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs 2024-11-23T05:55:07,693 INFO [RS:0;9abfc7c2569e:42619 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C42619%2C1732341304537:(num 1732341305244) 2024-11-23T05:55:07,693 DEBUG [RS:0;9abfc7c2569e:42619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:07,693 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:07,693 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:55:07,693 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.ChoreService(370): Chore service for: regionserver/9abfc7c2569e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-23T05:55:07,694 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T05:55:07,694 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T05:55:07,694 INFO [regionserver/9abfc7c2569e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:55:07,694 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T05:55:07,694 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:55:07,694 INFO [RS:0;9abfc7c2569e:42619 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42619 2024-11-23T05:55:07,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:07,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9abfc7c2569e,42619,1732341304537 2024-11-23T05:55:07,722 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:55:07,731 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9abfc7c2569e,42619,1732341304537] 2024-11-23T05:55:07,739 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9abfc7c2569e,42619,1732341304537 already deleted, retry=false 2024-11-23T05:55:07,739 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 9abfc7c2569e,42619,1732341304537 on 9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:07,745 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 9abfc7c2569e,42619,1732341304537, splitWal=true, meta=false 2024-11-23T05:55:07,748 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 9abfc7c2569e,42619,1732341304537, splitWal=true, meta=false 2024-11-23T05:55:07,748 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 9abfc7c2569e,42619,1732341304537 (carryingMeta=false) 9abfc7c2569e,42619,1732341304537/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@2676a798[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-23T05:55:07,749 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 9abfc7c2569e,42619,1732341304537 had 1 regions 2024-11-23T05:55:07,750 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 9abfc7c2569e,42619,1732341304537, splitWal=true, meta=false, isMeta: false 2024-11-23T05:55:07,751 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,42619,1732341304537-splitting 2024-11-23T05:55:07,751 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,42619,1732341304537-splitting dir is empty, no logs to split. 2024-11-23T05:55:07,751 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9abfc7c2569e,42619,1732341304537 WAL count=0, meta=false 2024-11-23T05:55:07,753 INFO [PEWorker-1 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,42619,1732341304537-splitting dir is empty, no logs to split. 2024-11-23T05:55:07,753 INFO [PEWorker-1 {}] master.SplitWALManager(105): 9abfc7c2569e,42619,1732341304537 WAL count=0, meta=false 2024-11-23T05:55:07,753 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 9abfc7c2569e,42619,1732341304537 WAL splitting is done? wals=0, meta=false 2024-11-23T05:55:07,754 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for 9abfc7c2569e,42619,1732341304537 failed, ignore...File file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,42619,1732341304537-splitting does not exist 2024-11-23T05:55:07,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN}] 2024-11-23T05:55:07,757 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN 2024-11-23T05:55:07,757 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-23T05:55:07,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:07,831 INFO [RS:0;9abfc7c2569e:42619 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:55:07,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42619-0x101678c1f050001, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:07,831 INFO [RS:0;9abfc7c2569e:42619 {}] regionserver.HRegionServer(1031): Exiting; stopping=9abfc7c2569e,42619,1732341304537; zookeeper connection closed. 2024-11-23T05:55:07,832 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@25018174 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@25018174 2024-11-23T05:55:07,908 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=aa214f0922457f1679e5487887d642e3, regionState=OPENING, regionLocation=9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:07,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN because future has completed 2024-11-23T05:55:07,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa214f0922457f1679e5487887d642e3, server=9abfc7c2569e,44239,1732341304577}] 2024-11-23T05:55:07,915 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:55:07,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:07,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:07,915 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:55:07,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:55:07,915 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:55:07,915 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T05:55:07,916 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:55:07,916 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36339 2024-11-23T05:55:07,918 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36339 connecting to ZooKeeper ensemble=127.0.0.1:51849 2024-11-23T05:55:07,918 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:07,918 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:07,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:363390x0, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:55:07,938 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36339-0x101678c1f050003 connected 2024-11-23T05:55:07,938 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-23T05:55:07,938 DEBUG [pool-297-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-23T05:55:07,940 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T05:55:07,940 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.cache 2024-11-23T05:55:07,941 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-23T05:55:07,941 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-11-23T05:55:07,941 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-23T05:55:07,941 INFO [Thread-201 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-11-23T05:55:07,941 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-23T05:55:07,942 INFO [Thread-201 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-11-23T05:55:07,943 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T05:55:07,943 INFO [Thread-201 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-11-23T05:55:07,944 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T05:55:07,946 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:55:07,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36339 2024-11-23T05:55:07,947 DEBUG [Thread-201 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-11-23T05:55:07,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36339 2024-11-23T05:55:07,947 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36339 2024-11-23T05:55:07,948 INFO [Thread-201 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-23T05:55:07,948 INFO [Thread-201 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-11-23T05:55:07,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36339 2024-11-23T05:55:07,948 INFO [Thread-201 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence is complete. 2024-11-23T05:55:07,948 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36339 2024-11-23T05:55:07,950 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(746): ClusterId : 3e36e304-43b7-4c14-b16f-bb4dc11783de 2024-11-23T05:55:07,950 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T05:55:07,957 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T05:55:07,957 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T05:55:07,965 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T05:55:07,965 DEBUG [RS:2;9abfc7c2569e:36339 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ad5cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:55:07,977 DEBUG [RS:2;9abfc7c2569e:36339 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;9abfc7c2569e:36339 2024-11-23T05:55:07,977 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T05:55:07,977 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T05:55:07,977 DEBUG [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T05:55:07,978 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(2659): reportForDuty to master=9abfc7c2569e,36109,1732341304400 with port=36339, startcode=1732341307915 2024-11-23T05:55:07,979 DEBUG [RS:2;9abfc7c2569e:36339 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T05:55:07,980 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60637, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T05:55:07,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36109 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:07,981 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36109 {}] master.ServerManager(517): Registering regionserver=9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:07,983 DEBUG [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a 2024-11-23T05:55:07,983 DEBUG [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-23T05:55:07,983 DEBUG [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T05:55:07,989 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:07,989 DEBUG [RS:2;9abfc7c2569e:36339 {}] zookeeper.ZKUtil(111): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:07,990 WARN [RS:2;9abfc7c2569e:36339 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:55:07,990 INFO [RS:2;9abfc7c2569e:36339 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:55:07,990 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9abfc7c2569e,36339,1732341307915] 2024-11-23T05:55:07,990 DEBUG [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:07,994 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T05:55:07,996 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T05:55:07,997 INFO [RS:2;9abfc7c2569e:36339 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T05:55:07,997 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:07,997 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T05:55:07,998 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T05:55:07,998 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,998 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,999 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,999 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,999 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:55:07,999 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:07,999 DEBUG [RS:2;9abfc7c2569e:36339 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:55:08,000 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,000 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,000 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,000 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,000 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,000 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36339,1732341307915-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:55:08,014 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T05:55:08,014 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,36339,1732341307915-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,015 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,015 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.Replication(171): 9abfc7c2569e,36339,1732341307915 started 2024-11-23T05:55:08,029 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:55:08,029 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(1482): Serving as 9abfc7c2569e,36339,1732341307915, RpcServer on 9abfc7c2569e/172.17.0.2:36339, sessionid=0x101678c1f050003 2024-11-23T05:55:08,029 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T05:55:08,029 DEBUG [RS:2;9abfc7c2569e:36339 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:08,029 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,36339,1732341307915' 2024-11-23T05:55:08,029 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T05:55:08,030 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T05:55:08,030 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T05:55:08,030 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T05:55:08,030 DEBUG [RS:2;9abfc7c2569e:36339 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:08,030 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,36339,1732341307915' 2024-11-23T05:55:08,030 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T05:55:08,031 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T05:55:08,031 DEBUG [RS:2;9abfc7c2569e:36339 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T05:55:08,031 INFO [RS:2;9abfc7c2569e:36339 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T05:55:08,031 INFO [RS:2;9abfc7c2569e:36339 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T05:55:08,067 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,067 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => aa214f0922457f1679e5487887d642e3, NAME => 'testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:55:08,067 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1732341305714 aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,067 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:55:08,068 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,068 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,068 INFO [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,069 INFO [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aa214f0922457f1679e5487887d642e3 columnFamilyName family 2024-11-23T05:55:08,069 DEBUG [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:55:08,071 DEBUG [StoreFileOpener-aa214f0922457f1679e5487887d642e3-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848, delay=959 ms 2024-11-23T05:55:08,072 DEBUG [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848 2024-11-23T05:55:08,072 INFO [StoreOpener-aa214f0922457f1679e5487887d642e3-1 {}] regionserver.HStore(327): Store=aa214f0922457f1679e5487887d642e3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:55:08,073 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,073 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,074 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,074 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,074 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,076 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,077 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened aa214f0922457f1679e5487887d642e3; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58771502, jitterRate=-0.12423637509346008}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-23T05:55:08,077 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,077 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for aa214f0922457f1679e5487887d642e3: Running coprocessor pre-open hook at 1732341308068Writing region info on filesystem at 1732341308068Initializing all the Stores at 1732341308068Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732341308068Cleaning up temporary data from old regions at 1732341308074 (+6 ms)Running coprocessor post-open hooks at 1732341308077 (+3 ms)Region opened successfully at 1732341308077 2024-11-23T05:55:08,078 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3., pid=9, masterSystemTime=1732341308064 2024-11-23T05:55:08,081 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,081 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,081 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=aa214f0922457f1679e5487887d642e3, regionState=OPEN, openSeqNum=10, regionLocation=9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:08,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure aa214f0922457f1679e5487887d642e3, server=9abfc7c2569e,44239,1732341304577 because future has completed 2024-11-23T05:55:08,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-23T05:55:08,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure aa214f0922457f1679e5487887d642e3, server=9abfc7c2569e,44239,1732341304577 in 173 msec 2024-11-23T05:55:08,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T05:55:08,089 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 9abfc7c2569e,42619,1732341304537 after splitting done 2024-11-23T05:55:08,089 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1732341305714, region=aa214f0922457f1679e5487887d642e3, ASSIGN in 332 msec 2024-11-23T05:55:08,091 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 9abfc7c2569e,42619,1732341304537, splitWal=true, meta=false in 349 msec 2024-11-23T05:55:08,132 INFO [RS:2;9abfc7c2569e:36339 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:55:08,133 INFO [RS:2;9abfc7c2569e:36339 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=9abfc7c2569e%2C36339%2C1732341307915, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,36339,1732341307915, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs, maxLogs=55 2024-11-23T05:55:08,149 INFO [RS:2;9abfc7c2569e:36339 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/WALs/9abfc7c2569e,36339,1732341307915/9abfc7c2569e%2C36339%2C1732341307915.1732341308134 2024-11-23T05:55:08,150 DEBUG [RS:2;9abfc7c2569e:36339 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-23T05:55:08,444 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T05:55:08,445 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T05:55:08,447 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732341305714 2024-11-23T05:55:08,447 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testBlockEvictionOnGracefulStop1732341305714 Metrics about Tables on a single HBase RegionServer 2024-11-23T05:55:08,448 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T05:55:08,448 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T05:55:08,448 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T05:55:08,448 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T05:55:08,529 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T05:55:08,529 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:55:08,530 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:08,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:08,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:08,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:55:08,531 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T05:55:08,531 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1190287876, stopped=false 2024-11-23T05:55:08,531 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9abfc7c2569e,36109,1732341304400 2024-11-23T05:55:08,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:08,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:08,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:55:08,581 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T05:55:08,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:08,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:08,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:08,581 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:55:08,581 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:08,582 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:08,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:08,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:08,582 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:55:08,582 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9abfc7c2569e,44239,1732341304577' ***** 2024-11-23T05:55:08,582 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T05:55:08,582 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9abfc7c2569e,36339,1732341307915' ***** 2024-11-23T05:55:08,582 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T05:55:08,582 INFO [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T05:55:08,583 INFO [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1738): Disabling cache 2024-11-23T05:55:08,583 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T05:55:08,583 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T05:55:08,583 INFO [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence 2024-11-23T05:55:08,583 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T05:55:08,583 INFO [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e2bcb26b-b670-ca6d-5cd3-4fe214f2ac6d/bucket.persistence 2024-11-23T05:55:08,583 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-23T05:55:08,583 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-23T05:55:08,583 INFO [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T05:55:08,583 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-23T05:55:08,583 INFO [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-11-23T05:55:08,583 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T05:55:08,584 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-23T05:55:08,593 DEBUG [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1487): Thread RS:1;9abfc7c2569e:44239 started persisting bucket cache to file 2024-11-23T05:55:08,594 DEBUG [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1487): Thread RS:2;9abfc7c2569e:36339 started persisting bucket cache to file 2024-11-23T05:55:08,594 DEBUG [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:08,594 DEBUG [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-11-23T05:55:08,594 DEBUG [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:08,594 DEBUG [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-23T05:55:08,596 DEBUG [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T05:55:08,597 DEBUG [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732341308594 2024-11-23T05:55:08,596 ERROR [RS:1;9abfc7c2569e:44239 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:329) ~[?:?] at java.io.BufferedReader.readLine(BufferedReader.java:396) ~[?:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1056) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:55:08,597 DEBUG [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-23T05:55:08,597 DEBUG [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1732341308594 2024-11-23T05:55:08,597 DEBUG [RS:1;9abfc7c2569e:44239 {}] bucket.BucketCache(1509): Thread RS:1;9abfc7c2569e:44239 finished persisting bucket cache to file, renaming 2024-11-23T05:55:08,597 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T05:55:08,597 INFO [RS:1;9abfc7c2569e:44239 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T05:55:08,597 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T05:55:08,597 INFO [RS:1;9abfc7c2569e:44239 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T05:55:08,597 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(3091): Received CLOSE for aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,598 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(959): stopping server 9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:08,598 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:55:08,598 INFO [RS:1;9abfc7c2569e:44239 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;9abfc7c2569e:44239. 2024-11-23T05:55:08,598 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T05:55:08,598 DEBUG [RS:1;9abfc7c2569e:44239 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:08,598 DEBUG [RS:1;9abfc7c2569e:44239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:08,598 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing aa214f0922457f1679e5487887d642e3, disabling compactions & flushes 2024-11-23T05:55:08,598 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,598 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T05:55:08,598 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,598 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T05:55:08,598 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T05:55:08,598 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. after waiting 0 ms 2024-11-23T05:55:08,598 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T05:55:08,598 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,599 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-23T05:55:08,599 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, aa214f0922457f1679e5487887d642e3=testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3.} 2024-11-23T05:55:08,599 DEBUG [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, aa214f0922457f1679e5487887d642e3 2024-11-23T05:55:08,599 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-23T05:55:08,599 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T05:55:08,599 DEBUG [StoreFileCloser-aa214f0922457f1679e5487887d642e3-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/family/fe992f6d17dc488caf32bd3e74606848 2024-11-23T05:55:08,599 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T05:55:08,599 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T05:55:08,599 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T05:55:08,599 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T05:55:08,599 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.96 KB heapSize=5.58 KB 2024-11-23T05:55:08,600 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T05:55:08,601 INFO [regionserver/9abfc7c2569e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:08,603 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/default/testBlockEvictionOnGracefulStop1732341305714/aa214f0922457f1679e5487887d642e3/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-23T05:55:08,603 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,603 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for aa214f0922457f1679e5487887d642e3: Waiting for close lock at 1732341308598Running coprocessor pre-close hooks at 1732341308598Disabling compacts and flushes for region at 1732341308598Disabling writes for close at 1732341308598Writing region close event to WAL at 1732341308599 (+1 ms)Running coprocessor post-close hooks at 1732341308603 (+4 ms)Closed at 1732341308603 2024-11-23T05:55:08,603 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3. 2024-11-23T05:55:08,607 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 615c190b74db479cbd89e125a1c5c3c1 2024-11-23T05:55:08,609 INFO [regionserver/9abfc7c2569e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:08,612 DEBUG [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1509): Thread RS:2;9abfc7c2569e:36339 finished persisting bucket cache to file, renaming 2024-11-23T05:55:08,612 WARN [RS:2;9abfc7c2569e:36339 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-23T05:55:08,612 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T05:55:08,612 INFO [RS:2;9abfc7c2569e:36339 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T05:55:08,612 INFO [RS:2;9abfc7c2569e:36339 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T05:55:08,612 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(959): stopping server 9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:08,612 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:55:08,612 INFO [RS:2;9abfc7c2569e:36339 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;9abfc7c2569e:36339. 2024-11-23T05:55:08,613 DEBUG [RS:2;9abfc7c2569e:36339 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:55:08,613 DEBUG [RS:2;9abfc7c2569e:36339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:08,613 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(976): stopping server 9abfc7c2569e,36339,1732341307915; all regions closed. 2024-11-23T05:55:08,615 DEBUG [RS:2;9abfc7c2569e:36339 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs 2024-11-23T05:55:08,615 INFO [RS:2;9abfc7c2569e:36339 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C36339%2C1732341307915:(num 1732341308134) 2024-11-23T05:55:08,615 DEBUG [RS:2;9abfc7c2569e:36339 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:08,615 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:08,615 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:55:08,615 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.ChoreService(370): Chore service for: regionserver/9abfc7c2569e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T05:55:08,615 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T05:55:08,615 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T05:55:08,615 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T05:55:08,616 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:55:08,616 INFO [RS:2;9abfc7c2569e:36339 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36339 2024-11-23T05:55:08,616 INFO [regionserver/9abfc7c2569e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:55:08,623 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/info/615c190b74db479cbd89e125a1c5c3c1 is 209, key is testBlockEvictionOnGracefulStop1732341305714,,1732341305719.aa214f0922457f1679e5487887d642e3./info:regioninfo/1732341308081/Put/seqid=0 2024-11-23T05:55:08,623 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.74 KB at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/info/615c190b74db479cbd89e125a1c5c3c1 2024-11-23T05:55:08,630 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3e148a316fa54fed9dc5e1881106d614 2024-11-23T05:55:08,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9abfc7c2569e,36339,1732341307915 2024-11-23T05:55:08,641 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:08,641 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:55:08,643 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/ns/3e148a316fa54fed9dc5e1881106d614 is 43, key is default/ns:d/1732341305684/Put/seqid=0 2024-11-23T05:55:08,644 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/ns/3e148a316fa54fed9dc5e1881106d614 2024-11-23T05:55:08,651 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on b066520ca969409097cf1cafa0603070 2024-11-23T05:55:08,656 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9abfc7c2569e,36339,1732341307915] 2024-11-23T05:55:08,663 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/table/b066520ca969409097cf1cafa0603070 is 80, key is testBlockEvictionOnGracefulStop1732341305714/table:state/1732341306097/Put/seqid=0 2024-11-23T05:55:08,664 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9abfc7c2569e,36339,1732341307915 already deleted, retry=false 2024-11-23T05:55:08,664 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9abfc7c2569e,36339,1732341307915 expired; onlineServers=1 2024-11-23T05:55:08,664 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=13 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/table/b066520ca969409097cf1cafa0603070 2024-11-23T05:55:08,666 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/info/615c190b74db479cbd89e125a1c5c3c1 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/615c190b74db479cbd89e125a1c5c3c1 2024-11-23T05:55:08,667 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/615c190b74db479cbd89e125a1c5c3c1, delay=960 ms 2024-11-23T05:55:08,668 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/615c190b74db479cbd89e125a1c5c3c1, entries=14, sequenceid=13, filesize=7.5 K 2024-11-23T05:55:08,669 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/ns/3e148a316fa54fed9dc5e1881106d614 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/3e148a316fa54fed9dc5e1881106d614 2024-11-23T05:55:08,670 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/3e148a316fa54fed9dc5e1881106d614, delay=938 ms 2024-11-23T05:55:08,671 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/3e148a316fa54fed9dc5e1881106d614, entries=2, sequenceid=13, filesize=5.0 K 2024-11-23T05:55:08,671 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/.tmp/table/b066520ca969409097cf1cafa0603070 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/b066520ca969409097cf1cafa0603070 2024-11-23T05:55:08,672 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/b066520ca969409097cf1cafa0603070, delay=942 ms 2024-11-23T05:55:08,673 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/b066520ca969409097cf1cafa0603070, entries=2, sequenceid=13, filesize=5.3 K 2024-11-23T05:55:08,674 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=13, compaction requested=false 2024-11-23T05:55:08,674 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/info/615c190b74db479cbd89e125a1c5c3c1 2024-11-23T05:55:08,675 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/ns/3e148a316fa54fed9dc5e1881106d614 2024-11-23T05:55:08,676 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/table/b066520ca969409097cf1cafa0603070 2024-11-23T05:55:08,679 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/data/hbase/meta/1588230740/recovered.edits/16.seqid, newMaxSeqId=16, maxSeqId=1 2024-11-23T05:55:08,679 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T05:55:08,679 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T05:55:08,680 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732341308599Running coprocessor pre-close hooks at 1732341308599Disabling compacts and flushes for region at 1732341308599Disabling writes for close at 1732341308599Obtaining lock to block concurrent updates at 1732341308599Preparing flush snapshotting stores in 1588230740 at 1732341308599Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3035, getHeapSize=5648, getOffHeapSize=0, getCellsCount=22 at 1732341308600 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732341308600Flushing 1588230740/info: creating writer at 1732341308600Flushing 1588230740/info: appending metadata at 1732341308622 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732341308622Flushing 1588230740/ns: creating writer at 1732341308625 (+3 ms)Flushing 1588230740/ns: appending metadata at 1732341308642 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732341308642Flushing 1588230740/table: creating writer at 1732341308646 (+4 ms)Flushing 1588230740/table: appending metadata at 1732341308663 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732341308663Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34fc8129: reopening flushed file at 1732341308666 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@abe00fd: reopening flushed file at 1732341308669 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a91c076: reopening flushed file at 1732341308671 (+2 ms)Finished flush of dataSize ~2.96 KB/3035, heapSize ~5.28 KB/5408, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=13, compaction requested=false at 1732341308674 (+3 ms)Writing region close event to WAL at 1732341308676 (+2 ms)Running coprocessor post-close hooks at 1732341308679 (+3 ms)Closed at 1732341308679 2024-11-23T05:55:08,680 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T05:55:08,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:08,756 INFO [RS:2;9abfc7c2569e:36339 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:55:08,756 INFO [RS:2;9abfc7c2569e:36339 {}] regionserver.HRegionServer(1031): Exiting; stopping=9abfc7c2569e,36339,1732341307915; zookeeper connection closed. 2024-11-23T05:55:08,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36339-0x101678c1f050003, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:08,757 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2d9023fa {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2d9023fa 2024-11-23T05:55:08,799 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(976): stopping server 9abfc7c2569e,44239,1732341304577; all regions closed. 2024-11-23T05:55:08,801 DEBUG [RS:1;9abfc7c2569e:44239 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs 2024-11-23T05:55:08,801 INFO [RS:1;9abfc7c2569e:44239 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C44239%2C1732341304577.meta:.meta(num 1732341305611) 2024-11-23T05:55:08,803 DEBUG [RS:1;9abfc7c2569e:44239 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/oldWALs 2024-11-23T05:55:08,803 INFO [RS:1;9abfc7c2569e:44239 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C44239%2C1732341304577:(num 1732341305244) 2024-11-23T05:55:08,803 DEBUG [RS:1;9abfc7c2569e:44239 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:55:08,803 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:55:08,803 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:55:08,804 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.ChoreService(370): Chore service for: regionserver/9abfc7c2569e:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T05:55:08,804 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:55:08,804 INFO [regionserver/9abfc7c2569e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:55:08,804 INFO [RS:1;9abfc7c2569e:44239 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44239 2024-11-23T05:55:08,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9abfc7c2569e,44239,1732341304577 2024-11-23T05:55:08,820 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:55:08,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:55:08,830 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9abfc7c2569e,44239,1732341304577] 2024-11-23T05:55:08,839 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9abfc7c2569e,44239,1732341304577 already deleted, retry=false 2024-11-23T05:55:08,839 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9abfc7c2569e,44239,1732341304577 expired; onlineServers=0 2024-11-23T05:55:08,839 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9abfc7c2569e,36109,1732341304400' ***** 2024-11-23T05:55:08,839 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T05:55:08,839 INFO [M:0;9abfc7c2569e:36109 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:55:08,839 INFO [M:0;9abfc7c2569e:36109 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:55:08,839 DEBUG [M:0;9abfc7c2569e:36109 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T05:55:08,840 DEBUG [M:0;9abfc7c2569e:36109 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T05:55:08,840 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T05:55:08,840 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732341304941 {}] cleaner.HFileCleaner(306): Exit Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732341304941,5,FailOnTimeoutGroup] 2024-11-23T05:55:08,840 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732341304942 {}] cleaner.HFileCleaner(306): Exit Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732341304942,5,FailOnTimeoutGroup] 2024-11-23T05:55:08,840 INFO [M:0;9abfc7c2569e:36109 {}] hbase.ChoreService(370): Chore service for: master/9abfc7c2569e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T05:55:08,840 INFO [M:0;9abfc7c2569e:36109 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:55:08,841 DEBUG [M:0;9abfc7c2569e:36109 {}] master.HMaster(1795): Stopping service threads 2024-11-23T05:55:08,841 INFO [M:0;9abfc7c2569e:36109 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T05:55:08,841 INFO [M:0;9abfc7c2569e:36109 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T05:55:08,842 INFO [M:0;9abfc7c2569e:36109 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T05:55:08,842 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T05:55:08,847 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T05:55:08,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:55:08,848 DEBUG [M:0;9abfc7c2569e:36109 {}] zookeeper.ZKUtil(347): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T05:55:08,848 WARN [M:0;9abfc7c2569e:36109 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T05:55:08,848 INFO [M:0;9abfc7c2569e:36109 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/.lastflushedseqids 2024-11-23T05:55:08,854 INFO [M:0;9abfc7c2569e:36109 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T05:55:08,854 INFO [M:0;9abfc7c2569e:36109 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T05:55:08,854 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T05:55:08,854 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:08,854 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:08,854 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T05:55:08,854 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:08,854 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=32.83 KB heapSize=41.08 KB 2024-11-23T05:55:08,862 DEBUG [M:0;9abfc7c2569e:36109 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d59dbf17e62e480d8ad509eaffa91d4f 2024-11-23T05:55:08,874 DEBUG [M:0;9abfc7c2569e:36109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d59dbf17e62e480d8ad509eaffa91d4f is 82, key is hbase:meta,,1/info:regioninfo/1732341305640/Put/seqid=0 2024-11-23T05:55:08,875 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d59dbf17e62e480d8ad509eaffa91d4f 2024-11-23T05:55:08,881 DEBUG [M:0;9abfc7c2569e:36109 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 40c7863c29024b9583ea73a4f75c8570 2024-11-23T05:55:08,893 DEBUG [M:0;9abfc7c2569e:36109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40c7863c29024b9583ea73a4f75c8570 is 782, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732341306101/Put/seqid=0 2024-11-23T05:55:08,894 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=32.09 KB at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40c7863c29024b9583ea73a4f75c8570 2024-11-23T05:55:08,901 DEBUG [M:0;9abfc7c2569e:36109 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 39c620def42847aab1019a8457b6986f 2024-11-23T05:55:08,913 DEBUG [M:0;9abfc7c2569e:36109 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39c620def42847aab1019a8457b6986f is 69, key is 9abfc7c2569e,36339,1732341307915/rs:state/1732341307981/Put/seqid=0 2024-11-23T05:55:08,914 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=83 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39c620def42847aab1019a8457b6986f 2024-11-23T05:55:08,915 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 39c620def42847aab1019a8457b6986f 2024-11-23T05:55:08,916 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/d59dbf17e62e480d8ad509eaffa91d4f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d59dbf17e62e480d8ad509eaffa91d4f 2024-11-23T05:55:08,917 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/d59dbf17e62e480d8ad509eaffa91d4f, entries=8, sequenceid=83, filesize=5.5 K 2024-11-23T05:55:08,917 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/40c7863c29024b9583ea73a4f75c8570 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/40c7863c29024b9583ea73a4f75c8570 2024-11-23T05:55:08,919 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/40c7863c29024b9583ea73a4f75c8570, entries=9, sequenceid=83, filesize=6.4 K 2024-11-23T05:55:08,919 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/39c620def42847aab1019a8457b6986f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39c620def42847aab1019a8457b6986f 2024-11-23T05:55:08,920 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 39c620def42847aab1019a8457b6986f 2024-11-23T05:55:08,920 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3cce99a8-6ada-82dd-cd54-5bd26b93c362/2d5dd90c-ee75-1ffe-1659-8adf4ad9231a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/39c620def42847aab1019a8457b6986f, entries=3, sequenceid=83, filesize=5.3 K 2024-11-23T05:55:08,921 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=83, compaction requested=false 2024-11-23T05:55:08,922 INFO [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:55:08,923 DEBUG [M:0;9abfc7c2569e:36109 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732341308854Disabling compacts and flushes for region at 1732341308854Disabling writes for close at 1732341308854Obtaining lock to block concurrent updates at 1732341308854Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732341308854Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=33614, getHeapSize=42000, getOffHeapSize=0, getCellsCount=98 at 1732341308855 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732341308855Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732341308855Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732341308874 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732341308874Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732341308876 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732341308893 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732341308893Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732341308896 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732341308912 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732341308912Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56bea608: reopening flushed file at 1732341308915 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@26ccf706: reopening flushed file at 1732341308917 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13fdaf92: reopening flushed file at 1732341308919 (+2 ms)Finished flush of dataSize ~32.83 KB/33614, heapSize ~40.78 KB/41760, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 67ms, sequenceid=83, compaction requested=false at 1732341308921 (+2 ms)Writing region close event to WAL at 1732341308922 (+1 ms)Closed at 1732341308922 2024-11-23T05:55:08,923 INFO [M:0;9abfc7c2569e:36109 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T05:55:08,924 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:55:08,924 INFO [M:0;9abfc7c2569e:36109 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36109 2024-11-23T05:55:08,924 INFO [M:0;9abfc7c2569e:36109 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:55:08,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:08,931 INFO [RS:1;9abfc7c2569e:44239 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:55:08,931 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44239-0x101678c1f050002, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:08,931 INFO [RS:1;9abfc7c2569e:44239 {}] regionserver.HRegionServer(1031): Exiting; stopping=9abfc7c2569e,44239,1732341304577; zookeeper connection closed. 2024-11-23T05:55:08,931 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@48156e6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@48156e6 2024-11-23T05:55:08,931 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-23T05:55:09,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:09,046 INFO [M:0;9abfc7c2569e:36109 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:55:09,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36109-0x101678c1f050000, quorum=127.0.0.1:51849, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:55:09,051 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T05:55:09,053 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-23T05:55:09,054 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:55:09,062 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=73 (was 51) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=342 (was 354), ProcessCount=11 (was 11), AvailableMemoryMB=7623 (was 7723)