2024-12-10 12:04:58,189 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 12:04:58,207 main DEBUG Took 0.016113 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-10 12:04:58,208 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-10 12:04:58,208 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-10 12:04:58,209 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-10 12:04:58,210 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,218 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-10 12:04:58,232 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,233 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,234 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,234 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,235 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,236 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,237 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,237 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,238 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,239 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,239 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,240 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,240 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,241 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,241 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,241 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,242 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,243 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,243 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,244 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,244 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-10 12:04:58,245 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,245 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-10 12:04:58,247 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-10 12:04:58,249 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-10 12:04:58,250 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-10 12:04:58,251 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-10 12:04:58,253 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-10 12:04:58,253 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-10 12:04:58,263 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-10 12:04:58,266 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-10 12:04:58,268 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-10 12:04:58,268 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-10 12:04:58,268 main DEBUG createAppenders(={Console}) 2024-12-10 12:04:58,269 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-12-10 12:04:58,269 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-12-10 12:04:58,270 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-12-10 12:04:58,270 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-10 12:04:58,271 main DEBUG OutputStream closed 2024-12-10 12:04:58,271 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-10 12:04:58,271 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-10 12:04:58,271 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-12-10 12:04:58,334 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-10 12:04:58,336 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-10 12:04:58,337 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-10 12:04:58,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-10 12:04:58,338 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-10 12:04:58,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-10 12:04:58,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-10 12:04:58,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-10 12:04:58,339 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-10 12:04:58,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-10 12:04:58,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-10 12:04:58,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-10 12:04:58,340 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-10 12:04:58,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-10 12:04:58,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-10 12:04:58,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-10 12:04:58,341 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-10 12:04:58,342 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-10 12:04:58,344 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10 12:04:58,344 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-12-10 12:04:58,344 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-10 12:04:58,345 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-12-10T12:04:58,356 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-12-10 12:04:58,358 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-10 12:04:58,359 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-10T12:04:58,561 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c 2024-12-10T12:04:58,581 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=94, ProcessCount=11, AvailableMemoryMB=6922 2024-12-10T12:04:58,682 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-10T12:04:58,795 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:04:58,812 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/cluster_e84f6c6b-54fa-017b-c77f-92ca99507fa4, deleteOnExit=true 2024-12-10T12:04:58,949 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/cluster_e84f6c6b-54fa-017b-c77f-92ca99507fa4/zookeeper_0, clientPort=58397, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/cluster_e84f6c6b-54fa-017b-c77f-92ca99507fa4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/cluster_e84f6c6b-54fa-017b-c77f-92ca99507fa4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T12:04:58,962 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58397 2024-12-10T12:04:58,962 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:04:59,040 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b with version=8 2024-12-10T12:04:59,040 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/hbase-staging 2024-12-10T12:04:59,122 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-10T12:04:59,352 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef751fafe6b1:0 server-side Connection retries=45 2024-12-10T12:04:59,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:04:59,361 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:04:59,368 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:04:59,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:04:59,369 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:04:59,496 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T12:04:59,577 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:04:59,620 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-10T12:04:59,629 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-10T12:04:59,633 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:04:59,666 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 110736 (auto-detected) 2024-12-10T12:04:59,667 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-10T12:04:59,691 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42475 2024-12-10T12:04:59,714 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42475 connecting to ZooKeeper ensemble=127.0.0.1:58397 2024-12-10T12:04:59,816 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424750x0, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:04:59,818 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42475-0x1000fab104a0000 connected 2024-12-10T12:04:59,897 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:04:59,898 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:04:59,906 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:04:59,910 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b, hbase.cluster.distributed=false 2024-12-10T12:04:59,937 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:04:59,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42475 2024-12-10T12:04:59,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42475 2024-12-10T12:04:59,943 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42475 2024-12-10T12:04:59,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42475 2024-12-10T12:04:59,944 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42475 2024-12-10T12:05:00,020 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef751fafe6b1:0 server-side Connection retries=45 2024-12-10T12:05:00,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:00,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:00,022 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:05:00,022 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:00,023 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:05:00,025 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T12:05:00,027 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:05:00,028 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41057 2024-12-10T12:05:00,030 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41057 connecting to ZooKeeper ensemble=127.0.0.1:58397 2024-12-10T12:05:00,030 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:00,031 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:00,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:410570x0, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:05:00,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:410570x0, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:00,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41057-0x1000fab104a0001 connected 2024-12-10T12:05:00,058 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T12:05:00,071 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.cache 2024-12-10T12:05:00,078 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T12:05:00,080 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T12:05:00,082 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T12:05:00,084 INFO [Thread-10 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T12:05:00,084 WARN [Thread-10 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T12:05:00,091 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T12:05:00,095 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T12:05:00,096 INFO [Thread-10 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.persistence is complete. 2024-12-10T12:05:00,097 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T12:05:00,101 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T12:05:00,107 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:05:00,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41057 2024-12-10T12:05:00,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41057 2024-12-10T12:05:00,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41057 2024-12-10T12:05:00,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41057 2024-12-10T12:05:00,110 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41057 2024-12-10T12:05:00,133 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef751fafe6b1:0 server-side Connection retries=45 2024-12-10T12:05:00,133 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:00,133 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:00,133 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:05:00,134 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:00,134 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:05:00,134 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T12:05:00,134 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:05:00,135 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44881 2024-12-10T12:05:00,138 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44881 connecting to ZooKeeper ensemble=127.0.0.1:58397 2024-12-10T12:05:00,138 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:00,139 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:00,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:448810x0, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:05:00,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:00,178 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44881-0x1000fab104a0002 connected 2024-12-10T12:05:00,179 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T12:05:00,180 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.cache 2024-12-10T12:05:00,181 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T12:05:00,181 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T12:05:00,181 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T12:05:00,182 INFO [Thread-11 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T12:05:00,182 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T12:05:00,182 WARN [Thread-11 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T12:05:00,183 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T12:05:00,183 INFO [Thread-11 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.persistence is complete. 2024-12-10T12:05:00,187 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T12:05:00,188 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T12:05:00,191 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:05:00,195 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44881 2024-12-10T12:05:00,196 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44881 2024-12-10T12:05:00,198 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44881 2024-12-10T12:05:00,207 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44881 2024-12-10T12:05:00,208 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44881 2024-12-10T12:05:00,232 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef751fafe6b1:42475 2024-12-10T12:05:00,235 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:00,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:00,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:00,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:00,270 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:00,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T12:05:00,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:00,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T12:05:00,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:00,303 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:00,304 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T12:05:00,306 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef751fafe6b1,42475,1733832299196 from backup master directory 2024-12-10T12:05:00,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:00,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:00,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:00,317 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:00,318 WARN [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:05:00,318 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:00,320 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-10T12:05:00,322 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-10T12:05:00,330 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/hbase.id] with ID: 9a197fea-8ffa-4948-9baa-a9812029bf09 2024-12-10T12:05:00,330 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.tmp/hbase.id 2024-12-10T12:05:00,335 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/hbase.id] 2024-12-10T12:05:00,372 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:00,377 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T12:05:00,395 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-12-10T12:05:00,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:00,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:00,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:00,434 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:05:00,436 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T12:05:00,478 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:05:00,486 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:00,545 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store 2024-12-10T12:05:00,569 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-10T12:05:00,572 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:00,574 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T12:05:00,575 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:00,575 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:00,577 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T12:05:00,577 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:00,578 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:00,578 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733832300574Disabling compacts and flushes for region at 1733832300574Disabling writes for close at 1733832300577 (+3 ms)Writing region close event to WAL at 1733832300578 (+1 ms)Closed at 1733832300578 2024-12-10T12:05:00,579 WARN [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/.initializing 2024-12-10T12:05:00,580 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/WALs/ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:00,592 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:05:00,605 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef751fafe6b1%2C42475%2C1733832299196, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/WALs/ef751fafe6b1,42475,1733832299196, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/oldWALs, maxLogs=10 2024-12-10T12:05:00,640 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/WALs/ef751fafe6b1,42475,1733832299196/ef751fafe6b1%2C42475%2C1733832299196.1733832300610 2024-12-10T12:05:00,640 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:00,641 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:00,641 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:00,644 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,645 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,688 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,724 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T12:05:00,729 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:00,732 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:00,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T12:05:00,738 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:00,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:00,739 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T12:05:00,747 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:00,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:00,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,754 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T12:05:00,754 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:00,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:00,755 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,760 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,760 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,764 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,765 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,769 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:05:00,772 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:00,780 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:05:00,781 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72160787, jitterRate=0.07527951896190643}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:05:00,790 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733832300657Initializing all the Stores at 1733832300657Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832300658 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832300660 (+2 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832300660Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832300661 (+1 ms)Cleaning up temporary data from old regions at 1733832300765 (+104 ms)Region opened successfully at 1733832300790 (+25 ms) 2024-12-10T12:05:00,792 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T12:05:00,832 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c34e25c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef751fafe6b1/172.17.0.2:0 2024-12-10T12:05:00,875 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T12:05:00,891 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T12:05:00,891 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T12:05:00,896 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T12:05:00,896 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-10T12:05:00,904 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-12-10T12:05:00,904 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T12:05:00,932 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T12:05:00,943 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T12:05:01,002 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T12:05:01,004 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T12:05:01,006 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T12:05:01,016 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T12:05:01,019 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T12:05:01,023 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T12:05:01,035 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T12:05:01,037 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T12:05:01,043 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T12:05:01,059 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T12:05:01,068 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T12:05:01,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:01,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:01,091 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:01,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,095 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef751fafe6b1,42475,1733832299196, sessionid=0x1000fab104a0000, setting cluster-up flag (Was=false) 2024-12-10T12:05:01,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,144 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T12:05:01,146 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:01,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,168 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,193 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T12:05:01,195 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:01,199 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T12:05:01,214 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(746): ClusterId : 9a197fea-8ffa-4948-9baa-a9812029bf09 2024-12-10T12:05:01,214 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(746): ClusterId : 9a197fea-8ffa-4948-9baa-a9812029bf09 2024-12-10T12:05:01,217 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T12:05:01,217 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T12:05:01,229 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T12:05:01,229 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T12:05:01,229 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T12:05:01,229 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T12:05:01,237 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T12:05:01,237 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T12:05:01,238 DEBUG [RS:0;ef751fafe6b1:41057 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156c3899, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef751fafe6b1/172.17.0.2:0 2024-12-10T12:05:01,238 DEBUG [RS:1;ef751fafe6b1:44881 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16785dd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef751fafe6b1/172.17.0.2:0 2024-12-10T12:05:01,251 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ef751fafe6b1:44881 2024-12-10T12:05:01,254 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T12:05:01,254 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T12:05:01,254 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T12:05:01,255 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef751fafe6b1:41057 2024-12-10T12:05:01,255 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T12:05:01,255 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T12:05:01,255 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T12:05:01,256 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef751fafe6b1,42475,1733832299196 with port=44881, startcode=1733832300132 2024-12-10T12:05:01,257 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef751fafe6b1,42475,1733832299196 with port=41057, startcode=1733832299989 2024-12-10T12:05:01,271 DEBUG [RS:0;ef751fafe6b1:41057 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T12:05:01,271 DEBUG [RS:1;ef751fafe6b1:44881 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T12:05:01,281 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:01,292 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T12:05:01,303 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T12:05:01,309 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef751fafe6b1,42475,1733832299196 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T12:05:01,318 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:01,318 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:01,319 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:01,319 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:01,319 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef751fafe6b1:0, corePoolSize=10, maxPoolSize=10 2024-12-10T12:05:01,319 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,319 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:05:01,319 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,324 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733832331324 2024-12-10T12:05:01,325 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:01,325 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T12:05:01,326 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T12:05:01,328 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T12:05:01,330 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:01,331 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T12:05:01,332 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T12:05:01,332 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T12:05:01,333 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T12:05:01,333 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T12:05:01,334 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,338 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T12:05:01,340 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T12:05:01,340 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45509, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T12:05:01,340 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60761, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T12:05:01,341 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T12:05:01,344 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T12:05:01,345 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T12:05:01,347 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.large.0-1733832301346,5,FailOnTimeoutGroup] 2024-12-10T12:05:01,348 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-10T12:05:01,350 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.small.0-1733832301347,5,FailOnTimeoutGroup] 2024-12-10T12:05:01,350 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,350 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T12:05:01,351 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,351 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,356 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:01,358 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T12:05:01,359 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(517): Registering regionserver=ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:01,359 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:01,366 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:01,367 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T12:05:01,367 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T12:05:01,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:01,373 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T12:05:01,378 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T12:05:01,378 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:01,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:01,379 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T12:05:01,382 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-10T12:05:01,383 WARN [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-10T12:05:01,389 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T12:05:01,389 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:01,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:01,390 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T12:05:01,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:01,392 DEBUG [RS:0;ef751fafe6b1:41057 {}] zookeeper.ZKUtil(111): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:01,392 WARN [RS:0;ef751fafe6b1:41057 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:05:01,393 INFO [RS:0;ef751fafe6b1:41057 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:01,393 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:01,395 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T12:05:01,395 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:01,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:01,396 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T12:05:01,396 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef751fafe6b1,41057,1733832299989] 2024-12-10T12:05:01,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T12:05:01,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:01,403 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:01,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T12:05:01,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:01,407 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:01,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T12:05:01,408 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T12:05:01,408 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:05:01,409 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T12:05:01,417 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:05:01,418 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72322660, jitterRate=0.07769161462783813}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:05:01,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733832301368Initializing all the Stores at 1733832301369 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832301369Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832301373 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832301373Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832301373Cleaning up temporary data from old regions at 1733832301408 (+35 ms)Region opened successfully at 1733832301422 (+14 ms) 2024-12-10T12:05:01,422 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T12:05:01,422 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T12:05:01,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T12:05:01,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T12:05:01,423 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T12:05:01,425 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:01,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733832301422Disabling compacts and flushes for region at 1733832301422Disabling writes for close at 1733832301423 (+1 ms)Writing region close event to WAL at 1733832301424 (+1 ms)Closed at 1733832301425 (+1 ms) 2024-12-10T12:05:01,427 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T12:05:01,428 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:01,428 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T12:05:01,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T12:05:01,441 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T12:05:01,443 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T12:05:01,446 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T12:05:01,453 INFO [RS:0;ef751fafe6b1:41057 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T12:05:01,453 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,455 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T12:05:01,459 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T12:05:01,460 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,461 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,461 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,461 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,461 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,461 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,461 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef751fafe6b1:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:05:01,461 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,462 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,462 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,462 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,462 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,462 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,462 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:01,462 DEBUG [RS:0;ef751fafe6b1:41057 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:01,464 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,464 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,465 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,465 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,465 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,465 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,41057,1733832299989-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:05:01,484 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef751fafe6b1,42475,1733832299196 with port=44881, startcode=1733832300132 2024-12-10T12:05:01,485 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T12:05:01,487 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,41057,1733832299989-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,488 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,488 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.Replication(171): ef751fafe6b1,41057,1733832299989 started 2024-12-10T12:05:01,488 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:01,489 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42475 {}] master.ServerManager(517): Registering regionserver=ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:01,491 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:01,492 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T12:05:01,492 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T12:05:01,506 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,507 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1482): Serving as ef751fafe6b1,41057,1733832299989, RpcServer on ef751fafe6b1/172.17.0.2:41057, sessionid=0x1000fab104a0001 2024-12-10T12:05:01,508 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T12:05:01,508 DEBUG [RS:0;ef751fafe6b1:41057 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:01,508 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,41057,1733832299989' 2024-12-10T12:05:01,508 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T12:05:01,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:01,536 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T12:05:01,537 DEBUG [RS:1;ef751fafe6b1:44881 {}] zookeeper.ZKUtil(111): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:01,537 WARN [RS:1;ef751fafe6b1:44881 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:05:01,537 INFO [RS:1;ef751fafe6b1:44881 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:01,537 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:01,538 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef751fafe6b1,44881,1733832300132] 2024-12-10T12:05:01,538 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T12:05:01,538 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T12:05:01,538 DEBUG [RS:0;ef751fafe6b1:41057 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:01,538 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,41057,1733832299989' 2024-12-10T12:05:01,538 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T12:05:01,539 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T12:05:01,540 DEBUG [RS:0;ef751fafe6b1:41057 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T12:05:01,540 INFO [RS:0;ef751fafe6b1:41057 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T12:05:01,540 INFO [RS:0;ef751fafe6b1:41057 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T12:05:01,543 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T12:05:01,548 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T12:05:01,549 INFO [RS:1;ef751fafe6b1:44881 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T12:05:01,549 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,549 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T12:05:01,550 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T12:05:01,550 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,551 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,551 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,551 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,551 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,551 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,551 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef751fafe6b1:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:05:01,552 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,552 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,552 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,552 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,552 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,553 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:01,553 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:01,553 DEBUG [RS:1;ef751fafe6b1:44881 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:01,554 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,554 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,554 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,554 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,554 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,554 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,44881,1733832300132-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:05:01,574 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T12:05:01,574 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,44881,1733832300132-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,575 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,575 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.Replication(171): ef751fafe6b1,44881,1733832300132 started 2024-12-10T12:05:01,594 WARN [ef751fafe6b1:42475 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T12:05:01,598 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:01,598 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1482): Serving as ef751fafe6b1,44881,1733832300132, RpcServer on ef751fafe6b1/172.17.0.2:44881, sessionid=0x1000fab104a0002 2024-12-10T12:05:01,598 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T12:05:01,598 DEBUG [RS:1;ef751fafe6b1:44881 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:01,598 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,44881,1733832300132' 2024-12-10T12:05:01,599 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T12:05:01,600 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T12:05:01,600 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T12:05:01,600 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T12:05:01,600 DEBUG [RS:1;ef751fafe6b1:44881 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:01,600 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,44881,1733832300132' 2024-12-10T12:05:01,601 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T12:05:01,601 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T12:05:01,602 DEBUG [RS:1;ef751fafe6b1:44881 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T12:05:01,602 INFO [RS:1;ef751fafe6b1:44881 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T12:05:01,602 INFO [RS:1;ef751fafe6b1:44881 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T12:05:01,645 INFO [RS:0;ef751fafe6b1:41057 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:05:01,646 INFO [RS:0;ef751fafe6b1:41057 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C41057%2C1733832299989, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41057,1733832299989, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:01,668 INFO [RS:0;ef751fafe6b1:41057 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41057,1733832299989/ef751fafe6b1%2C41057%2C1733832299989.1733832301649 2024-12-10T12:05:01,668 DEBUG [RS:0;ef751fafe6b1:41057 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:01,703 INFO [RS:1;ef751fafe6b1:44881 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:05:01,705 INFO [RS:1;ef751fafe6b1:44881 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C44881%2C1733832300132, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,44881,1733832300132, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:01,729 INFO [RS:1;ef751fafe6b1:44881 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,44881,1733832300132/ef751fafe6b1%2C44881%2C1733832300132.1733832301707 2024-12-10T12:05:01,729 DEBUG [RS:1;ef751fafe6b1:44881 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:01,846 DEBUG [ef751fafe6b1:42475 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T12:05:01,855 DEBUG [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(204): Hosts are {ef751fafe6b1=0} racks are {/default-rack=0} 2024-12-10T12:05:01,858 DEBUG [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T12:05:01,858 DEBUG [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T12:05:01,858 DEBUG [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T12:05:01,858 DEBUG [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T12:05:01,858 INFO [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T12:05:01,858 INFO [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T12:05:01,858 DEBUG [ef751fafe6b1:42475 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T12:05:01,864 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:01,868 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef751fafe6b1,44881,1733832300132, state=OPENING 2024-12-10T12:05:01,900 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T12:05:01,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:01,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:01,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:01,911 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:01,913 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T12:05:01,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef751fafe6b1,44881,1733832300132}] 2024-12-10T12:05:02,083 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T12:05:02,086 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41361, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T12:05:02,093 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T12:05:02,094 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:02,095 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T12:05:02,096 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C44881%2C1733832300132.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,44881,1733832300132, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:02,113 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,44881,1733832300132/ef751fafe6b1%2C44881%2C1733832300132.meta.1733832302098.meta 2024-12-10T12:05:02,113 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:02,114 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:02,115 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T12:05:02,117 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T12:05:02,121 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T12:05:02,125 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T12:05:02,126 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:02,126 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T12:05:02,126 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T12:05:02,128 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T12:05:02,129 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T12:05:02,129 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:02,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:02,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T12:05:02,130 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T12:05:02,130 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:02,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:02,131 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T12:05:02,131 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T12:05:02,131 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:02,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:02,132 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T12:05:02,132 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T12:05:02,133 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:02,133 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:02,133 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T12:05:02,134 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:02,134 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:02,135 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T12:05:02,135 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T12:05:02,135 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:05:02,136 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T12:05:02,137 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72084336, jitterRate=0.07414031028747559}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:05:02,137 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T12:05:02,138 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733832302127Writing region info on filesystem at 1733832302127Initializing all the Stores at 1733832302127Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832302127Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832302128 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832302128Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832302128Cleaning up temporary data from old regions at 1733832302135 (+7 ms)Running coprocessor post-open hooks at 1733832302137 (+2 ms)Region opened successfully at 1733832302138 (+1 ms) 2024-12-10T12:05:02,144 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733832302077 2024-12-10T12:05:02,152 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T12:05:02,153 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T12:05:02,154 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:02,156 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef751fafe6b1,44881,1733832300132, state=OPEN 2024-12-10T12:05:02,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:02,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:02,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:02,194 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:02,194 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:02,194 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:02,194 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:02,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T12:05:02,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef751fafe6b1,44881,1733832300132 in 280 msec 2024-12-10T12:05:02,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T12:05:02,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 764 msec 2024-12-10T12:05:02,203 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:02,203 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T12:05:02,225 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T12:05:02,227 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,44881,1733832300132, seqNum=-1] 2024-12-10T12:05:02,244 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:05:02,246 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43989, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:05:02,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0330 sec 2024-12-10T12:05:02,265 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733832302265, completionTime=-1 2024-12-10T12:05:02,268 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T12:05:02,268 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T12:05:02,297 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T12:05:02,297 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733832362297 2024-12-10T12:05:02,298 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733832422298 2024-12-10T12:05:02,298 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-12-10T12:05:02,300 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T12:05:02,306 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,42475,1733832299196-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:02,306 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,42475,1733832299196-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:02,307 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,42475,1733832299196-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:02,308 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef751fafe6b1:42475, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:02,308 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:02,308 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:02,313 DEBUG [master/ef751fafe6b1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T12:05:02,335 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.016sec 2024-12-10T12:05:02,336 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T12:05:02,337 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T12:05:02,338 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T12:05:02,338 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T12:05:02,338 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T12:05:02,339 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,42475,1733832299196-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:05:02,339 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,42475,1733832299196-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T12:05:02,340 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T12:05:02,341 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T12:05:02,341 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,42475,1733832299196-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:02,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@319823c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:05:02,426 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-10T12:05:02,426 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-10T12:05:02,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef751fafe6b1,42475,-1 for getting cluster id 2024-12-10T12:05:02,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T12:05:02,442 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '9a197fea-8ffa-4948-9baa-a9812029bf09' 2024-12-10T12:05:02,444 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T12:05:02,445 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "9a197fea-8ffa-4948-9baa-a9812029bf09" 2024-12-10T12:05:02,445 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558e824e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:05:02,445 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef751fafe6b1,42475,-1] 2024-12-10T12:05:02,447 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T12:05:02,449 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:02,450 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T12:05:02,453 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e12cb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:05:02,453 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T12:05:02,460 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,44881,1733832300132, seqNum=-1] 2024-12-10T12:05:02,461 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:05:02,463 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:05:02,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:02,495 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T12:05:02,503 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:02,507 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2d5554d3 2024-12-10T12:05:02,508 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:05:02,510 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41376, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:05:02,515 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1733832302492', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:05:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 2024-12-10T12:05:02,522 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:05:02,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1733832302492" procId is: 4 2024-12-10T12:05:02,524 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:02,526 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:05:02,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:02,541 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 9d4bd76d8def45c5d22425e6a326a402, NAME => 'testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1733832302492', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:02,547 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:02,547 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(1722): Closing 9d4bd76d8def45c5d22425e6a326a402, disabling compactions & flushes 2024-12-10T12:05:02,547 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:02,547 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:02,547 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. after waiting 0 ms 2024-12-10T12:05:02,547 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:02,547 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:02,547 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1733832302492-pool-0 {}] regionserver.HRegion(1676): Region close journal for 9d4bd76d8def45c5d22425e6a326a402: Waiting for close lock at 1733832302547Disabling compacts and flushes for region at 1733832302547Disabling writes for close at 1733832302547Writing region close event to WAL at 1733832302547Closed at 1733832302547 2024-12-10T12:05:02,549 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:05:02,555 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733832302550"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733832302550"}]},"ts":"1733832302550"} 2024-12-10T12:05:02,559 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T12:05:02,560 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:05:02,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733832302492","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733832302561"}]},"ts":"1733832302561"} 2024-12-10T12:05:02,566 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733832302492, state=ENABLING in hbase:meta 2024-12-10T12:05:02,567 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ef751fafe6b1=0} racks are {/default-rack=0} 2024-12-10T12:05:02,568 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T12:05:02,568 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T12:05:02,568 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T12:05:02,568 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T12:05:02,568 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T12:05:02,568 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T12:05:02,568 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T12:05:02,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, ASSIGN}] 2024-12-10T12:05:02,571 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, ASSIGN 2024-12-10T12:05:02,573 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, ASSIGN; state=OFFLINE, location=ef751fafe6b1,44881,1733832300132; forceNewPlan=false, retain=false 2024-12-10T12:05:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:02,725 INFO [ef751fafe6b1:42475 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T12:05:02,726 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9d4bd76d8def45c5d22425e6a326a402, regionState=OPENING, regionLocation=ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:02,729 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, ASSIGN because future has completed 2024-12-10T12:05:02,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,44881,1733832300132}] 2024-12-10T12:05:02,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:02,885 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:02,886 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d4bd76d8def45c5d22425e6a326a402, NAME => 'testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:02,886 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733832302492 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,886 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:02,886 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,886 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,887 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,890 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d4bd76d8def45c5d22425e6a326a402 columnFamilyName family 2024-12-10T12:05:02,891 DEBUG [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:02,891 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.HStore(327): Store=9d4bd76d8def45c5d22425e6a326a402/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:02,891 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,892 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,892 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,892 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,892 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,893 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,899 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:05:02,900 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 9d4bd76d8def45c5d22425e6a326a402; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73745534, jitterRate=0.09889408946037292}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T12:05:02,900 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:02,901 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 9d4bd76d8def45c5d22425e6a326a402: Running coprocessor pre-open hook at 1733832302887Writing region info on filesystem at 1733832302887Initializing all the Stores at 1733832302887Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832302887Cleaning up temporary data from old regions at 1733832302892 (+5 ms)Running coprocessor post-open hooks at 1733832302900 (+8 ms)Region opened successfully at 1733832302901 (+1 ms) 2024-12-10T12:05:02,902 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402., pid=6, masterSystemTime=1733832302882 2024-12-10T12:05:02,905 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:02,905 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:02,906 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=9d4bd76d8def45c5d22425e6a326a402, regionState=OPEN, openSeqNum=2, regionLocation=ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:02,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,44881,1733832300132 because future has completed 2024-12-10T12:05:02,913 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T12:05:02,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,44881,1733832300132 in 181 msec 2024-12-10T12:05:02,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T12:05:02,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, ASSIGN in 343 msec 2024-12-10T12:05:02,916 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:05:02,916 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1733832302492","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733832302916"}]},"ts":"1733832302916"} 2024-12-10T12:05:02,920 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1733832302492, state=ENABLED in hbase:meta 2024-12-10T12:05:02,921 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:05:02,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1733832302492 in 404 msec 2024-12-10T12:05:03,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:03,158 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1733832302492 get assigned. Timeout = 60000ms 2024-12-10T12:05:03,158 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1733832302492 completed 2024-12-10T12:05:03,160 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T12:05:03,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1733832302492 assigned to meta. Checking AM states. 2024-12-10T12:05:03,165 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T12:05:03,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1733832302492 assigned. 2024-12-10T12:05:03,173 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1733832302492', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402., hostname=ef751fafe6b1,44881,1733832300132, seqNum=2] 2024-12-10T12:05:03,182 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9d4bd76d8def45c5d22425e6a326a402 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T12:05:03,204 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:03,236 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/.tmp/family/7572a4e58d734145bc7f2d6325e9d822 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T12:05:03,241 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/.tmp/family/7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:03,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/.tmp/family/7572a4e58d734145bc7f2d6325e9d822 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:03,271 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822, delay=933 ms 2024-12-10T12:05:03,273 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T12:05:03,278 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 9d4bd76d8def45c5d22425e6a326a402 in 95ms, sequenceid=6, compaction requested=false 2024-12-10T12:05:03,279 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1733832302492' 2024-12-10T12:05:03,281 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9d4bd76d8def45c5d22425e6a326a402: 2024-12-10T12:05:04,206 DEBUG [hfile-prefetch-1733832303271 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7572a4e58d734145bc7f2d6325e9d822_0. 2024-12-10T12:05:04,210 DEBUG [hfile-prefetch-1733832303271 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 7572a4e58d734145bc7f2d6325e9d822_117. 2024-12-10T12:05:04,211 DEBUG [hfile-prefetch-1733832303271 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822, with total blocks 2, and data blocks 1 2024-12-10T12:05:04,211 DEBUG [hfile-prefetch-1733832303271 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T12:05:04,211 DEBUG [hfile-prefetch-1733832303271 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T12:05:04,212 DEBUG [hfile-prefetch-1733832303271 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822. Will sleep 100ms and try the verification again. 2024-12-10T12:05:04,293 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=9d4bd76d8def45c5d22425e6a326a402, source=ef751fafe6b1,44881,1733832300132, destination=ef751fafe6b1,41057,1733832299989, warming up region on ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:04,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T12:05:04,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=9d4bd76d8def45c5d22425e6a326a402, source=ef751fafe6b1,44881,1733832300132, destination=ef751fafe6b1,41057,1733832299989, running balancer 2024-12-10T12:05:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, REOPEN/MOVE 2024-12-10T12:05:04,297 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, REOPEN/MOVE 2024-12-10T12:05:04,297 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48495, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T12:05:04,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,299 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d4bd76d8def45c5d22425e6a326a402, regionState=CLOSING, regionLocation=ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:04,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(7855): Warmup {ENCODED => 9d4bd76d8def45c5d22425e6a326a402, NAME => 'testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:04,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:04,300 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,301 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d4bd76d8def45c5d22425e6a326a402 columnFamilyName family 2024-12-10T12:05:04,301 DEBUG [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:04,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, REOPEN/MOVE because future has completed 2024-12-10T12:05:04,303 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-12-10T12:05:04,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,44881,1733832300132}] 2024-12-10T12:05:04,305 DEBUG [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:04,305 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.HStore(327): Store=9d4bd76d8def45c5d22425e6a326a402/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(1722): Closing 9d4bd76d8def45c5d22425e6a326a402, disabling compactions & flushes 2024-12-10T12:05:04,306 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. after waiting 0 ms 2024-12-10T12:05:04,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,307 DEBUG [StoreFileCloser-9d4bd76d8def45c5d22425e6a326a402-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:04,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41057 {}] regionserver.HRegion(1676): Region close journal for 9d4bd76d8def45c5d22425e6a326a402: Waiting for close lock at 1733832304306Disabling compacts and flushes for region at 1733832304306Disabling writes for close at 1733832304306Writing region close event to WAL at 1733832304308 (+2 ms)Closed at 1733832304308 2024-12-10T12:05:04,309 WARN [hfile-prefetch-1733832303271 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-10T12:05:04,309 DEBUG [hfile-prefetch-1733832303271 {}] hfile.PrefetchExecutor(121): Prefetch completed for 7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42475 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:04,387 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:04,402 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:04,402 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832304387 2024-12-10T12:05:04,402 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T12:05:04,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-10T12:05:04,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733832304387 2024-12-10T12:05:04,405 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T12:05:04,405 WARN [bucket-cache-persister {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T12:05:04,461 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,462 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-12-10T12:05:04,462 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 9d4bd76d8def45c5d22425e6a326a402, disabling compactions & flushes 2024-12-10T12:05:04,462 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,462 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,462 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. after waiting 0 ms 2024-12-10T12:05:04,462 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,464 DEBUG [StoreFileCloser-9d4bd76d8def45c5d22425e6a326a402-family-1 {}] bucket.BucketCache(1863): found 1 blocks for file 7572a4e58d734145bc7f2d6325e9d822, starting offset: 0, end offset: 9223372036854775807 2024-12-10T12:05:04,465 DEBUG [StoreFileCloser-9d4bd76d8def45c5d22425e6a326a402-family-1 {}] bucket.BucketCache(856): removed key 7572a4e58d734145bc7f2d6325e9d822_0 from back map with offset lock 2094080 in the evict process 2024-12-10T12:05:04,469 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T12:05:04,470 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,470 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 9d4bd76d8def45c5d22425e6a326a402: Waiting for close lock at 1733832304462Running coprocessor pre-close hooks at 1733832304462Disabling compacts and flushes for region at 1733832304462Disabling writes for close at 1733832304462Writing region close event to WAL at 1733832304465 (+3 ms)Running coprocessor post-close hooks at 1733832304469 (+4 ms)Closed at 1733832304470 (+1 ms) 2024-12-10T12:05:04,470 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 9d4bd76d8def45c5d22425e6a326a402 move to ef751fafe6b1,41057,1733832299989 record at close sequenceid=6 2024-12-10T12:05:04,473 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,474 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d4bd76d8def45c5d22425e6a326a402, regionState=CLOSED 2024-12-10T12:05:04,476 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,44881,1733832300132 because future has completed 2024-12-10T12:05:04,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T12:05:04,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,44881,1733832300132 in 173 msec 2024-12-10T12:05:04,480 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, REOPEN/MOVE; state=CLOSED, location=ef751fafe6b1,41057,1733832299989; forceNewPlan=false, retain=false 2024-12-10T12:05:04,502 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T12:05:04,502 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T12:05:04,502 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:04,502 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:04,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:04,505 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T12:05:04,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832304502 2024-12-10T12:05:04,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T12:05:04,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T12:05:04,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:04,505 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:04,508 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:04,508 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832304505 2024-12-10T12:05:04,508 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T12:05:04,630 INFO [ef751fafe6b1:42475 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T12:05:04,631 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d4bd76d8def45c5d22425e6a326a402, regionState=OPENING, regionLocation=ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:04,634 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, REOPEN/MOVE because future has completed 2024-12-10T12:05:04,635 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,41057,1733832299989}] 2024-12-10T12:05:04,789 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,790 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 9d4bd76d8def45c5d22425e6a326a402, NAME => 'testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:04,790 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1733832302492 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,790 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:04,790 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,790 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,791 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,792 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9d4bd76d8def45c5d22425e6a326a402 columnFamilyName family 2024-12-10T12:05:04,792 DEBUG [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:04,794 DEBUG [StoreFileOpener-9d4bd76d8def45c5d22425e6a326a402-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822, delay=945 ms 2024-12-10T12:05:04,795 DEBUG [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:04,795 INFO [StoreOpener-9d4bd76d8def45c5d22425e6a326a402-1 {}] regionserver.HStore(327): Store=9d4bd76d8def45c5d22425e6a326a402/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:04,796 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,796 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,797 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,797 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,797 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,801 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,801 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 9d4bd76d8def45c5d22425e6a326a402; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59057527, jitterRate=-0.11997427046298981}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T12:05:04,801 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,802 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 9d4bd76d8def45c5d22425e6a326a402: Running coprocessor pre-open hook at 1733832304790Writing region info on filesystem at 1733832304790Initializing all the Stores at 1733832304791 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832304791Cleaning up temporary data from old regions at 1733832304797 (+6 ms)Running coprocessor post-open hooks at 1733832304801 (+4 ms)Region opened successfully at 1733832304802 (+1 ms) 2024-12-10T12:05:04,803 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402., pid=9, masterSystemTime=1733832304787 2024-12-10T12:05:04,806 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,806 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,806 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=9d4bd76d8def45c5d22425e6a326a402, regionState=OPEN, openSeqNum=10, regionLocation=ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:04,809 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,41057,1733832299989 because future has completed 2024-12-10T12:05:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-10T12:05:04,813 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 9d4bd76d8def45c5d22425e6a326a402, server=ef751fafe6b1,41057,1733832299989 in 177 msec 2024-12-10T12:05:04,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1733832302492, region=9d4bd76d8def45c5d22425e6a326a402, REOPEN/MOVE in 517 msec 2024-12-10T12:05:04,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T12:05:04,850 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T12:05:04,850 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:04,854 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:04,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:04,855 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T12:05:04,855 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T12:05:04,855 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=587878705, stopped=false 2024-12-10T12:05:04,855 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef751fafe6b1,42475,1733832299196 2024-12-10T12:05:04,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:04,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:04,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:04,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:04,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:04,901 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:04,901 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T12:05:04,901 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T12:05:04,902 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:04,902 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:04,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:04,902 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef751fafe6b1,41057,1733832299989' ***** 2024-12-10T12:05:04,902 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:04,902 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T12:05:04,903 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:04,903 INFO [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T12:05:04,903 INFO [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.persistence 2024-12-10T12:05:04,903 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef751fafe6b1,44881,1733832300132' ***** 2024-12-10T12:05:04,903 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T12:05:04,903 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T12:05:04,903 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T12:05:04,903 INFO [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T12:05:04,903 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T12:05:04,903 INFO [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/bucket.persistence 2024-12-10T12:05:04,903 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T12:05:04,903 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T12:05:04,903 INFO [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T12:05:04,904 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T12:05:04,904 INFO [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T12:05:04,904 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T12:05:04,904 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T12:05:04,914 DEBUG [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1487): Thread RS:0;ef751fafe6b1:41057 started persisting bucket cache to file 2024-12-10T12:05:04,914 DEBUG [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1487): Thread RS:1;ef751fafe6b1:44881 started persisting bucket cache to file 2024-12-10T12:05:04,914 DEBUG [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:04,914 DEBUG [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:04,914 DEBUG [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:04,914 DEBUG [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:04,916 DEBUG [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:04,916 DEBUG [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:04,916 DEBUG [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832304914 2024-12-10T12:05:04,917 DEBUG [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832304914 2024-12-10T12:05:04,917 DEBUG [RS:1;ef751fafe6b1:44881 {}] bucket.BucketCache(1509): Thread RS:1;ef751fafe6b1:44881 finished persisting bucket cache to file, renaming 2024-12-10T12:05:04,917 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T12:05:04,917 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T12:05:04,917 INFO [RS:1;ef751fafe6b1:44881 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T12:05:04,917 INFO [RS:1;ef751fafe6b1:44881 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T12:05:04,917 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(959): stopping server ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:04,917 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T12:05:04,918 INFO [RS:1;ef751fafe6b1:44881 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ef751fafe6b1:44881. 2024-12-10T12:05:04,918 DEBUG [RS:1;ef751fafe6b1:44881 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:04,918 DEBUG [RS:1;ef751fafe6b1:44881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:04,918 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T12:05:04,918 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T12:05:04,918 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T12:05:04,918 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T12:05:04,919 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T12:05:04,919 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T12:05:04,919 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T12:05:04,919 DEBUG [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T12:05:04,919 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T12:05:04,919 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T12:05:04,920 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T12:05:04,920 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T12:05:04,920 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T12:05:04,920 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-12-10T12:05:04,929 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ae5fc5a9fa464ea5b57f85eb42f1c988 2024-12-10T12:05:04,930 DEBUG [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1509): Thread RS:0;ef751fafe6b1:41057 finished persisting bucket cache to file, renaming 2024-12-10T12:05:04,930 WARN [RS:0;ef751fafe6b1:41057 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T12:05:04,930 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T12:05:04,930 INFO [RS:0;ef751fafe6b1:41057 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T12:05:04,930 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T12:05:04,930 INFO [RS:0;ef751fafe6b1:41057 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T12:05:04,930 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(3091): Received CLOSE for 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,931 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(959): stopping server ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:04,931 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T12:05:04,931 INFO [RS:0;ef751fafe6b1:41057 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef751fafe6b1:41057. 2024-12-10T12:05:04,931 DEBUG [RS:0;ef751fafe6b1:41057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:04,931 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T12:05:04,931 DEBUG [RS:0;ef751fafe6b1:41057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:04,931 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T12:05:04,931 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9d4bd76d8def45c5d22425e6a326a402, disabling compactions & flushes 2024-12-10T12:05:04,931 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1325): Online Regions={9d4bd76d8def45c5d22425e6a326a402=testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402.} 2024-12-10T12:05:04,931 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,931 DEBUG [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1351): Waiting on 9d4bd76d8def45c5d22425e6a326a402 2024-12-10T12:05:04,931 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,931 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. after waiting 0 ms 2024-12-10T12:05:04,931 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,932 DEBUG [StoreFileCloser-9d4bd76d8def45c5d22425e6a326a402-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/family/7572a4e58d734145bc7f2d6325e9d822 2024-12-10T12:05:04,937 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnRegionMove1733832302492/9d4bd76d8def45c5d22425e6a326a402/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-12-10T12:05:04,937 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,937 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9d4bd76d8def45c5d22425e6a326a402: Waiting for close lock at 1733832304931Running coprocessor pre-close hooks at 1733832304931Disabling compacts and flushes for region at 1733832304931Disabling writes for close at 1733832304931Writing region close event to WAL at 1733832304933 (+2 ms)Running coprocessor post-close hooks at 1733832304937 (+4 ms)Closed at 1733832304937 2024-12-10T12:05:04,937 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402. 2024-12-10T12:05:04,945 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/ae5fc5a9fa464ea5b57f85eb42f1c988 is 205, key is testBlockEvictionOnRegionMove1733832302492,,1733832302511.9d4bd76d8def45c5d22425e6a326a402./info:regioninfo/1733832304806/Put/seqid=0 2024-12-10T12:05:04,946 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/ae5fc5a9fa464ea5b57f85eb42f1c988 2024-12-10T12:05:04,957 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 79efc8a3475247d28d30d1848d24a1f7 2024-12-10T12:05:04,957 INFO [regionserver/ef751fafe6b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:04,968 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/ns/79efc8a3475247d28d30d1848d24a1f7 is 43, key is default/ns:d/1733832302251/Put/seqid=0 2024-12-10T12:05:04,969 INFO [regionserver/ef751fafe6b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:04,969 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/ns/79efc8a3475247d28d30d1848d24a1f7 2024-12-10T12:05:04,977 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 307d5126ffe84618abcc6965b1055884 2024-12-10T12:05:04,993 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/table/307d5126ffe84618abcc6965b1055884 is 78, key is testBlockEvictionOnRegionMove1733832302492/table:state/1733832302916/Put/seqid=0 2024-12-10T12:05:04,994 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/table/307d5126ffe84618abcc6965b1055884 2024-12-10T12:05:04,997 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/ae5fc5a9fa464ea5b57f85eb42f1c988 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/ae5fc5a9fa464ea5b57f85eb42f1c988 2024-12-10T12:05:04,999 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/ae5fc5a9fa464ea5b57f85eb42f1c988, delay=987 ms 2024-12-10T12:05:05,000 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/ae5fc5a9fa464ea5b57f85eb42f1c988, entries=15, sequenceid=15, filesize=7.6 K 2024-12-10T12:05:05,001 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/ns/79efc8a3475247d28d30d1848d24a1f7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/79efc8a3475247d28d30d1848d24a1f7 2024-12-10T12:05:05,002 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/79efc8a3475247d28d30d1848d24a1f7, delay=900 ms 2024-12-10T12:05:05,004 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/79efc8a3475247d28d30d1848d24a1f7, entries=2, sequenceid=15, filesize=5.0 K 2024-12-10T12:05:05,005 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/table/307d5126ffe84618abcc6965b1055884 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/307d5126ffe84618abcc6965b1055884 2024-12-10T12:05:05,006 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/307d5126ffe84618abcc6965b1055884, delay=948 ms 2024-12-10T12:05:05,008 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/307d5126ffe84618abcc6965b1055884, entries=2, sequenceid=15, filesize=5.3 K 2024-12-10T12:05:05,009 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false 2024-12-10T12:05:05,009 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-10T12:05:05,011 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/ae5fc5a9fa464ea5b57f85eb42f1c988 2024-12-10T12:05:05,012 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/79efc8a3475247d28d30d1848d24a1f7 2024-12-10T12:05:05,013 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/307d5126ffe84618abcc6965b1055884 2024-12-10T12:05:05,017 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-10T12:05:05,018 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T12:05:05,018 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:05,018 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733832304919Running coprocessor pre-close hooks at 1733832304919Disabling compacts and flushes for region at 1733832304919Disabling writes for close at 1733832304920 (+1 ms)Obtaining lock to block concurrent updates at 1733832304920Preparing flush snapshotting stores in 1588230740 at 1733832304920Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1733832304920Flushing stores of hbase:meta,,1.1588230740 at 1733832304921 (+1 ms)Flushing 1588230740/info: creating writer at 1733832304921Flushing 1588230740/info: appending metadata at 1733832304944 (+23 ms)Flushing 1588230740/info: closing flushed file at 1733832304944Flushing 1588230740/ns: creating writer at 1733832304950 (+6 ms)Flushing 1588230740/ns: appending metadata at 1733832304968 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1733832304968Flushing 1588230740/table: creating writer at 1733832304971 (+3 ms)Flushing 1588230740/table: appending metadata at 1733832304992 (+21 ms)Flushing 1588230740/table: closing flushed file at 1733832304992Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bffd5ef: reopening flushed file at 1733832304997 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@324c768c: reopening flushed file at 1733832305001 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e13598f: reopening flushed file at 1733832305004 (+3 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 89ms, sequenceid=15, compaction requested=false at 1733832305009 (+5 ms)Writing region close event to WAL at 1733832305013 (+4 ms)Running coprocessor post-close hooks at 1733832305017 (+4 ms)Closed at 1733832305018 (+1 ms) 2024-12-10T12:05:05,018 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:05,120 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(976): stopping server ef751fafe6b1,44881,1733832300132; all regions closed. 2024-12-10T12:05:05,124 DEBUG [RS:1;ef751fafe6b1:44881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:05,124 INFO [RS:1;ef751fafe6b1:44881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C44881%2C1733832300132.meta:.meta(num 1733832302098) 2024-12-10T12:05:05,126 DEBUG [RS:1;ef751fafe6b1:44881 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:05,126 INFO [RS:1;ef751fafe6b1:44881 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C44881%2C1733832300132:(num 1733832301707) 2024-12-10T12:05:05,126 DEBUG [RS:1;ef751fafe6b1:44881 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:05,126 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:05,126 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T12:05:05,127 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.ChoreService(370): Chore service for: regionserver/ef751fafe6b1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T12:05:05,127 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T12:05:05,127 INFO [regionserver/ef751fafe6b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T12:05:05,127 INFO [RS:1;ef751fafe6b1:44881 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44881 2024-12-10T12:05:05,131 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(976): stopping server ef751fafe6b1,41057,1733832299989; all regions closed. 2024-12-10T12:05:05,133 DEBUG [RS:0;ef751fafe6b1:41057 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C41057%2C1733832299989:(num 1733832301649) 2024-12-10T12:05:05,134 DEBUG [RS:0;ef751fafe6b1:41057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.ChoreService(370): Chore service for: regionserver/ef751fafe6b1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T12:05:05,134 INFO [regionserver/ef751fafe6b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T12:05:05,134 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T12:05:05,135 INFO [RS:0;ef751fafe6b1:41057 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41057 2024-12-10T12:05:05,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef751fafe6b1,44881,1733832300132 2024-12-10T12:05:05,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:05,141 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T12:05:05,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef751fafe6b1,41057,1733832299989 2024-12-10T12:05:05,151 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T12:05:05,151 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef751fafe6b1,44881,1733832300132] 2024-12-10T12:05:05,151 ERROR [Time-limited test-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$249/0x00007f4040477b20@10ffd2f7 rejected from java.util.concurrent.ThreadPoolExecutor@55fd4e8c[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 14] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-10T12:05:05,168 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef751fafe6b1,44881,1733832300132 already deleted, retry=false 2024-12-10T12:05:05,168 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef751fafe6b1,44881,1733832300132 expired; onlineServers=1 2024-12-10T12:05:05,168 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef751fafe6b1,41057,1733832299989] 2024-12-10T12:05:05,176 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef751fafe6b1,41057,1733832299989 already deleted, retry=false 2024-12-10T12:05:05,176 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef751fafe6b1,41057,1733832299989 expired; onlineServers=0 2024-12-10T12:05:05,176 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef751fafe6b1,42475,1733832299196' ***** 2024-12-10T12:05:05,176 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T12:05:05,176 INFO [M:0;ef751fafe6b1:42475 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T12:05:05,176 INFO [M:0;ef751fafe6b1:42475 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T12:05:05,176 DEBUG [M:0;ef751fafe6b1:42475 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T12:05:05,177 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T12:05:05,177 DEBUG [M:0;ef751fafe6b1:42475 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T12:05:05,177 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.small.0-1733832301347 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.small.0-1733832301347,5,FailOnTimeoutGroup] 2024-12-10T12:05:05,177 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.large.0-1733832301346 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.large.0-1733832301346,5,FailOnTimeoutGroup] 2024-12-10T12:05:05,177 INFO [M:0;ef751fafe6b1:42475 {}] hbase.ChoreService(370): Chore service for: master/ef751fafe6b1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T12:05:05,177 INFO [M:0;ef751fafe6b1:42475 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T12:05:05,177 DEBUG [M:0;ef751fafe6b1:42475 {}] master.HMaster(1795): Stopping service threads 2024-12-10T12:05:05,177 INFO [M:0;ef751fafe6b1:42475 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T12:05:05,177 INFO [M:0;ef751fafe6b1:42475 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T12:05:05,178 INFO [M:0;ef751fafe6b1:42475 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T12:05:05,178 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T12:05:05,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T12:05:05,184 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,185 DEBUG [M:0;ef751fafe6b1:42475 {}] zookeeper.ZKUtil(347): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T12:05:05,185 WARN [M:0;ef751fafe6b1:42475 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T12:05:05,185 INFO [M:0;ef751fafe6b1:42475 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.lastflushedseqids 2024-12-10T12:05:05,193 INFO [M:0;ef751fafe6b1:42475 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T12:05:05,194 INFO [M:0;ef751fafe6b1:42475 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T12:05:05,194 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T12:05:05,194 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,194 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,194 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T12:05:05,194 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,194 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-12-10T12:05:05,202 DEBUG [M:0;ef751fafe6b1:42475 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6726f3cfd84247a8b391cade8cee90a3 2024-12-10T12:05:05,214 DEBUG [M:0;ef751fafe6b1:42475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6726f3cfd84247a8b391cade8cee90a3 is 82, key is hbase:meta,,1/info:regioninfo/1733832302153/Put/seqid=0 2024-12-10T12:05:05,214 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6726f3cfd84247a8b391cade8cee90a3 2024-12-10T12:05:05,221 DEBUG [M:0;ef751fafe6b1:42475 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f3655f5f9ca64eb9b25a77be27a39503 2024-12-10T12:05:05,233 DEBUG [M:0;ef751fafe6b1:42475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3655f5f9ca64eb9b25a77be27a39503 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733832302922/Put/seqid=0 2024-12-10T12:05:05,234 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3655f5f9ca64eb9b25a77be27a39503 2024-12-10T12:05:05,241 DEBUG [M:0;ef751fafe6b1:42475 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 07b02eecbb7b48e5947935cf3d77e769 2024-12-10T12:05:05,251 DEBUG [M:0;ef751fafe6b1:42475 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07b02eecbb7b48e5947935cf3d77e769 is 69, key is ef751fafe6b1,41057,1733832299989/rs:state/1733832301361/Put/seqid=0 2024-12-10T12:05:05,252 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07b02eecbb7b48e5947935cf3d77e769 2024-12-10T12:05:05,253 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/6726f3cfd84247a8b391cade8cee90a3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6726f3cfd84247a8b391cade8cee90a3 2024-12-10T12:05:05,255 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/6726f3cfd84247a8b391cade8cee90a3, entries=8, sequenceid=84, filesize=5.5 K 2024-12-10T12:05:05,255 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3655f5f9ca64eb9b25a77be27a39503 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f3655f5f9ca64eb9b25a77be27a39503 2024-12-10T12:05:05,257 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f3655f5f9ca64eb9b25a77be27a39503, entries=9, sequenceid=84, filesize=6.5 K 2024-12-10T12:05:05,257 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/07b02eecbb7b48e5947935cf3d77e769 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07b02eecbb7b48e5947935cf3d77e769 2024-12-10T12:05:05,258 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/07b02eecbb7b48e5947935cf3d77e769, entries=2, sequenceid=84, filesize=5.1 K 2024-12-10T12:05:05,259 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 65ms, sequenceid=84, compaction requested=false 2024-12-10T12:05:05,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:05,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44881-0x1000fab104a0002, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:05,260 INFO [RS:1;ef751fafe6b1:44881 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T12:05:05,261 INFO [RS:1;ef751fafe6b1:44881 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef751fafe6b1,44881,1733832300132; zookeeper connection closed. 2024-12-10T12:05:05,261 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5c5adb83 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5c5adb83 2024-12-10T12:05:05,261 INFO [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,262 DEBUG [M:0;ef751fafe6b1:42475 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733832305194Disabling compacts and flushes for region at 1733832305194Disabling writes for close at 1733832305194Obtaining lock to block concurrent updates at 1733832305194Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733832305194Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1733832305195 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733832305195Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733832305195Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733832305213 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733832305213Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733832305216 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733832305233 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733832305233Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733832305235 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733832305251 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733832305251Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60d10f2f: reopening flushed file at 1733832305253 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d39f40: reopening flushed file at 1733832305255 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2be31822: reopening flushed file at 1733832305257 (+2 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 65ms, sequenceid=84, compaction requested=false at 1733832305259 (+2 ms)Writing region close event to WAL at 1733832305261 (+2 ms)Closed at 1733832305261 2024-12-10T12:05:05,264 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T12:05:05,264 INFO [M:0;ef751fafe6b1:42475 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T12:05:05,264 INFO [M:0;ef751fafe6b1:42475 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42475 2024-12-10T12:05:05,264 INFO [M:0;ef751fafe6b1:42475 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T12:05:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:05,268 INFO [RS:0;ef751fafe6b1:41057 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T12:05:05,268 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41057-0x1000fab104a0001, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:05,268 INFO [RS:0;ef751fafe6b1:41057 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef751fafe6b1,41057,1733832299989; zookeeper connection closed. 2024-12-10T12:05:05,268 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@e6f3251 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@e6f3251 2024-12-10T12:05:05,269 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-12-10T12:05:05,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:05,393 INFO [M:0;ef751fafe6b1:42475 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T12:05:05,393 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42475-0x1000fab104a0000, quorum=127.0.0.1:58397, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:05,397 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T12:05:05,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T12:05:05,422 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,430 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/ef751fafe6b1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ef751fafe6b1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733832304999 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733832303271 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/ef751fafe6b1:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/ef751fafe6b1:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: hfile-prefetch-1733832305002 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1733832304794 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=141 (was 94) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6541 (was 6922) 2024-12-10T12:05:05,435 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=141, ProcessCount=11, AvailableMemoryMB=6541 2024-12-10T12:05:05,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/hadoop-log-dir so I do NOT create it in target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e 2024-12-10T12:05:05,436 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/hadoop-tmp-dir so I do NOT create it in target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e 2024-12-10T12:05:05,436 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,438 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/cluster_fe865236-0689-8bc2-99d4-ac40b8f6ddaf, deleteOnExit=true 2024-12-10T12:05:05,441 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/cluster_fe865236-0689-8bc2-99d4-ac40b8f6ddaf/zookeeper_0, clientPort=59363, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/cluster_fe865236-0689-8bc2-99d4-ac40b8f6ddaf/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/cluster_fe865236-0689-8bc2-99d4-ac40b8f6ddaf/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-10T12:05:05,442 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59363 2024-12-10T12:05:05,453 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b with version=8 2024-12-10T12:05:05,453 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/hbase-staging 2024-12-10T12:05:05,456 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef751fafe6b1:0 server-side Connection retries=45 2024-12-10T12:05:05,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,456 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:05:05,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,456 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:05:05,456 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-10T12:05:05,457 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:05:05,457 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39461 2024-12-10T12:05:05,458 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39461 connecting to ZooKeeper ensemble=127.0.0.1:59363 2024-12-10T12:05:05,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:394610x0, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:05:05,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39461-0x1000fab29bd0000 connected 2024-12-10T12:05:05,560 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,560 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,561 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:05,562 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b, hbase.cluster.distributed=false 2024-12-10T12:05:05,564 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:05:05,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39461 2024-12-10T12:05:05,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39461 2024-12-10T12:05:05,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39461 2024-12-10T12:05:05,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39461 2024-12-10T12:05:05,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39461 2024-12-10T12:05:05,581 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef751fafe6b1:0 server-side Connection retries=45 2024-12-10T12:05:05,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,581 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:05:05,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:05:05,581 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T12:05:05,582 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:05:05,582 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41493 2024-12-10T12:05:05,584 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41493 connecting to ZooKeeper ensemble=127.0.0.1:59363 2024-12-10T12:05:05,584 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,584 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414930x0, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:05:05,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41493-0x1000fab29bd0001 connected 2024-12-10T12:05:05,594 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:05,594 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T12:05:05,595 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.cache 2024-12-10T12:05:05,595 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T12:05:05,595 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T12:05:05,595 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T12:05:05,597 INFO [Thread-120 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T12:05:05,597 WARN [Thread-120 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T12:05:05,597 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T12:05:05,597 INFO [Thread-120 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T12:05:05,597 INFO [Thread-120 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence is complete. 2024-12-10T12:05:05,599 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T12:05:05,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T12:05:05,601 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:05:05,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41493 2024-12-10T12:05:05,603 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41493 2024-12-10T12:05:05,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41493 2024-12-10T12:05:05,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41493 2024-12-10T12:05:05,607 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41493 2024-12-10T12:05:05,620 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef751fafe6b1:0 server-side Connection retries=45 2024-12-10T12:05:05,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,620 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:05:05,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:05,620 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:05:05,621 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T12:05:05,621 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:05:05,621 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38979 2024-12-10T12:05:05,623 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38979 connecting to ZooKeeper ensemble=127.0.0.1:59363 2024-12-10T12:05:05,623 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,623 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389790x0, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:05:05,635 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38979-0x1000fab29bd0002 connected 2024-12-10T12:05:05,636 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:05,636 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T12:05:05,636 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.cache 2024-12-10T12:05:05,636 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T12:05:05,637 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T12:05:05,637 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T12:05:05,637 INFO [Thread-121 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T12:05:05,637 WARN [Thread-121 {}] bucket.BucketCache(1533): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-12-10T12:05:05,637 INFO [Thread-121 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T12:05:05,637 INFO [Thread-121 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence is complete. 2024-12-10T12:05:05,639 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T12:05:05,639 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T12:05:05,640 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-10T12:05:05,643 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:05:05,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38979 2024-12-10T12:05:05,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38979 2024-12-10T12:05:05,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38979 2024-12-10T12:05:05,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38979 2024-12-10T12:05:05,653 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38979 2024-12-10T12:05:05,663 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef751fafe6b1:39461 2024-12-10T12:05:05,664 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:05,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:05,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:05,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:05,676 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:05,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T12:05:05,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-10T12:05:05,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,686 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T12:05:05,686 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef751fafe6b1,39461,1733832305455 from backup master directory 2024-12-10T12:05:05,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:05,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:05,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:05,693 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-10T12:05:05,693 WARN [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:05:05,693 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:05,695 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/hbase.id] with ID: a4555d43-9006-4544-83d6-816eee74586d 2024-12-10T12:05:05,695 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.tmp/hbase.id 2024-12-10T12:05:05,698 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/hbase.id] 2024-12-10T12:05:05,733 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:05,733 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-10T12:05:05,735 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-12-10T12:05:05,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,743 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,747 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:05:05,748 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-10T12:05:05,748 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:05,763 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store 2024-12-10T12:05:05,769 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:05,769 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T12:05:05,769 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,769 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,769 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T12:05:05,770 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,770 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:05,770 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733832305769Disabling compacts and flushes for region at 1733832305769Disabling writes for close at 1733832305769Writing region close event to WAL at 1733832305770 (+1 ms)Closed at 1733832305770 2024-12-10T12:05:05,770 WARN [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/.initializing 2024-12-10T12:05:05,770 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/WALs/ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:05,775 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:05:05,778 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef751fafe6b1%2C39461%2C1733832305455, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/WALs/ef751fafe6b1,39461,1733832305455, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/oldWALs, maxLogs=10 2024-12-10T12:05:05,793 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/WALs/ef751fafe6b1,39461,1733832305455/ef751fafe6b1%2C39461%2C1733832305455.1733832305778 2024-12-10T12:05:05,794 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:05,794 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:05,794 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:05,794 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,794 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,795 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,800 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-10T12:05:05,800 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:05,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:05,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-10T12:05:05,804 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:05,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:05,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-10T12:05:05,808 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:05,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:05,808 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-10T12:05:05,811 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:05,811 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:05,811 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,812 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,812 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,812 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,812 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,813 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:05:05,813 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-10T12:05:05,818 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:05:05,819 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71912019, jitterRate=0.07157258689403534}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:05:05,819 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733832305795Initializing all the Stores at 1733832305795Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832305795Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832305795Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832305795Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832305795Cleaning up temporary data from old regions at 1733832305812 (+17 ms)Region opened successfully at 1733832305819 (+7 ms) 2024-12-10T12:05:05,820 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-10T12:05:05,823 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f71b00d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef751fafe6b1/172.17.0.2:0 2024-12-10T12:05:05,824 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-10T12:05:05,824 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-10T12:05:05,824 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-10T12:05:05,824 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-10T12:05:05,824 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-12-10T12:05:05,825 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-12-10T12:05:05,825 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-10T12:05:05,826 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-10T12:05:05,827 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-10T12:05:05,843 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-10T12:05:05,843 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-10T12:05:05,844 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-10T12:05:05,851 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-10T12:05:05,852 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-10T12:05:05,853 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-10T12:05:05,860 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-10T12:05:05,861 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-10T12:05:05,868 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-10T12:05:05,871 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-10T12:05:05,876 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-10T12:05:05,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:05,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:05,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:05,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,886 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef751fafe6b1,39461,1733832305455, sessionid=0x1000fab29bd0000, setting cluster-up flag (Was=false) 2024-12-10T12:05:05,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,927 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-10T12:05:05,928 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:05,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,943 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:05,968 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-10T12:05:05,970 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:05,970 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-10T12:05:05,971 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:05,972 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-10T12:05:05,972 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-10T12:05:05,972 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef751fafe6b1,39461,1733832305455 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-10T12:05:05,974 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:05,974 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:05,974 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:05,974 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=5, maxPoolSize=5 2024-12-10T12:05:05,975 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef751fafe6b1:0, corePoolSize=10, maxPoolSize=10 2024-12-10T12:05:05,975 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:05,975 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:05:05,975 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:05,976 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733832335976 2024-12-10T12:05:05,976 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:05,976 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-10T12:05:05,976 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-10T12:05:05,977 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-10T12:05:05,977 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-10T12:05:05,977 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-10T12:05:05,977 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-10T12:05:05,977 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:05,977 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-10T12:05:05,977 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:05,977 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-10T12:05:05,977 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-10T12:05:05,978 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-10T12:05:05,978 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-10T12:05:05,978 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-10T12:05:05,978 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-10T12:05:05,978 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.large.0-1733832305978,5,FailOnTimeoutGroup] 2024-12-10T12:05:05,978 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.small.0-1733832305978,5,FailOnTimeoutGroup] 2024-12-10T12:05:05,978 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:05,979 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-10T12:05:05,979 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:05,979 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:05,996 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-10T12:05:05,996 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:06,002 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:06,003 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T12:05:06,011 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T12:05:06,011 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,011 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T12:05:06,015 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T12:05:06,015 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,015 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T12:05:06,019 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T12:05:06,019 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,019 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T12:05:06,023 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T12:05:06,023 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,023 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T12:05:06,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:06,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:06,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T12:05:06,024 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T12:05:06,024 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:05:06,025 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T12:05:06,030 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:05:06,030 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72503888, jitterRate=0.08039212226867676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:05:06,031 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733832306002Initializing all the Stores at 1733832306002Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832306002Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832306002Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832306002Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832306002Cleaning up temporary data from old regions at 1733832306024 (+22 ms)Region opened successfully at 1733832306031 (+7 ms) 2024-12-10T12:05:06,031 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T12:05:06,031 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T12:05:06,031 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T12:05:06,031 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T12:05:06,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T12:05:06,032 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:06,032 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733832306031Disabling compacts and flushes for region at 1733832306031Disabling writes for close at 1733832306031Writing region close event to WAL at 1733832306032 (+1 ms)Closed at 1733832306032 2024-12-10T12:05:06,033 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:06,033 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-10T12:05:06,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T12:05:06,034 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T12:05:06,035 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-10T12:05:06,055 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(746): ClusterId : a4555d43-9006-4544-83d6-816eee74586d 2024-12-10T12:05:06,055 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(746): ClusterId : a4555d43-9006-4544-83d6-816eee74586d 2024-12-10T12:05:06,055 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T12:05:06,055 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T12:05:06,078 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T12:05:06,078 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T12:05:06,078 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T12:05:06,078 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T12:05:06,086 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T12:05:06,086 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T12:05:06,086 DEBUG [RS:0;ef751fafe6b1:41493 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cbe94ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef751fafe6b1/172.17.0.2:0 2024-12-10T12:05:06,086 DEBUG [RS:1;ef751fafe6b1:38979 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a0364d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef751fafe6b1/172.17.0.2:0 2024-12-10T12:05:06,096 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef751fafe6b1:41493 2024-12-10T12:05:06,096 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T12:05:06,096 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T12:05:06,096 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T12:05:06,097 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;ef751fafe6b1:38979 2024-12-10T12:05:06,097 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T12:05:06,097 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T12:05:06,097 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T12:05:06,097 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef751fafe6b1,39461,1733832305455 with port=41493, startcode=1733832305580 2024-12-10T12:05:06,098 DEBUG [RS:0;ef751fafe6b1:41493 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T12:05:06,098 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef751fafe6b1,39461,1733832305455 with port=38979, startcode=1733832305620 2024-12-10T12:05:06,098 DEBUG [RS:1;ef751fafe6b1:38979 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T12:05:06,100 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58717, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T12:05:06,100 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37383, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T12:05:06,100 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39461 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:06,101 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39461 {}] master.ServerManager(517): Registering regionserver=ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:06,102 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39461 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,102 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39461 {}] master.ServerManager(517): Registering regionserver=ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,102 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:06,102 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T12:05:06,102 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T12:05:06,103 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:06,103 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T12:05:06,103 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T12:05:06,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:06,140 DEBUG [RS:0;ef751fafe6b1:41493 {}] zookeeper.ZKUtil(111): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:06,140 WARN [RS:0;ef751fafe6b1:41493 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:05:06,140 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef751fafe6b1,41493,1733832305580] 2024-12-10T12:05:06,140 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef751fafe6b1,38979,1733832305620] 2024-12-10T12:05:06,140 INFO [RS:0;ef751fafe6b1:41493 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:06,140 DEBUG [RS:1;ef751fafe6b1:38979 {}] zookeeper.ZKUtil(111): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,140 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:06,140 WARN [RS:1;ef751fafe6b1:38979 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:05:06,141 INFO [RS:1;ef751fafe6b1:38979 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:06,141 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,145 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T12:05:06,145 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T12:05:06,147 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T12:05:06,148 INFO [RS:1;ef751fafe6b1:38979 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T12:05:06,148 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,148 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T12:05:06,148 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T12:05:06,148 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T12:05:06,148 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,149 INFO [RS:0;ef751fafe6b1:41493 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef751fafe6b1:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:06,149 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,149 DEBUG [RS:1;ef751fafe6b1:38979 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef751fafe6b1:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:06,150 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:06,150 DEBUG [RS:0;ef751fafe6b1:41493 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:06,150 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,38979,1733832305620-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:05:06,151 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,151 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,151 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,151 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,151 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,151 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,41493,1733832305580-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:05:06,164 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T12:05:06,164 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,38979,1733832305620-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,164 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,164 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.Replication(171): ef751fafe6b1,38979,1733832305620 started 2024-12-10T12:05:06,168 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T12:05:06,168 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,41493,1733832305580-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,168 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,169 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.Replication(171): ef751fafe6b1,41493,1733832305580 started 2024-12-10T12:05:06,177 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,177 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1482): Serving as ef751fafe6b1,38979,1733832305620, RpcServer on ef751fafe6b1/172.17.0.2:38979, sessionid=0x1000fab29bd0002 2024-12-10T12:05:06,177 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T12:05:06,177 DEBUG [RS:1;ef751fafe6b1:38979 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,178 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,38979,1733832305620' 2024-12-10T12:05:06,178 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T12:05:06,178 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T12:05:06,179 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T12:05:06,179 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T12:05:06,179 DEBUG [RS:1;ef751fafe6b1:38979 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,179 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,38979,1733832305620' 2024-12-10T12:05:06,179 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T12:05:06,180 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T12:05:06,180 DEBUG [RS:1;ef751fafe6b1:38979 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T12:05:06,180 INFO [RS:1;ef751fafe6b1:38979 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T12:05:06,180 INFO [RS:1;ef751fafe6b1:38979 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T12:05:06,185 WARN [ef751fafe6b1:39461 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-10T12:05:06,189 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,189 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1482): Serving as ef751fafe6b1,41493,1733832305580, RpcServer on ef751fafe6b1/172.17.0.2:41493, sessionid=0x1000fab29bd0001 2024-12-10T12:05:06,189 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T12:05:06,189 DEBUG [RS:0;ef751fafe6b1:41493 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:06,189 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,41493,1733832305580' 2024-12-10T12:05:06,189 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T12:05:06,190 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T12:05:06,191 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T12:05:06,191 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T12:05:06,191 DEBUG [RS:0;ef751fafe6b1:41493 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:06,191 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,41493,1733832305580' 2024-12-10T12:05:06,191 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T12:05:06,192 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T12:05:06,192 DEBUG [RS:0;ef751fafe6b1:41493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T12:05:06,192 INFO [RS:0;ef751fafe6b1:41493 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T12:05:06,192 INFO [RS:0;ef751fafe6b1:41493 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T12:05:06,281 INFO [RS:1;ef751fafe6b1:38979 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:05:06,282 INFO [RS:1;ef751fafe6b1:38979 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C38979%2C1733832305620, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:06,293 INFO [RS:0;ef751fafe6b1:41493 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:05:06,293 INFO [RS:0;ef751fafe6b1:41493 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C41493%2C1733832305580, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41493,1733832305580, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:06,300 INFO [RS:1;ef751fafe6b1:38979 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620/ef751fafe6b1%2C38979%2C1733832305620.1733832306283 2024-12-10T12:05:06,300 DEBUG [RS:1;ef751fafe6b1:38979 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:06,309 INFO [RS:0;ef751fafe6b1:41493 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41493,1733832305580/ef751fafe6b1%2C41493%2C1733832305580.1733832306294 2024-12-10T12:05:06,309 DEBUG [RS:0;ef751fafe6b1:41493 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:06,436 DEBUG [ef751fafe6b1:39461 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-10T12:05:06,436 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(204): Hosts are {ef751fafe6b1=0} racks are {/default-rack=0} 2024-12-10T12:05:06,437 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T12:05:06,437 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T12:05:06,437 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T12:05:06,437 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T12:05:06,437 INFO [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T12:05:06,437 INFO [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T12:05:06,437 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T12:05:06,438 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,438 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef751fafe6b1,38979,1733832305620, state=OPENING 2024-12-10T12:05:06,460 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-10T12:05:06,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:06,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:06,516 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:06,517 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T12:05:06,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:06,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:06,517 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:06,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef751fafe6b1,38979,1733832305620}] 2024-12-10T12:05:06,669 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T12:05:06,671 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46381, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T12:05:06,673 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T12:05:06,673 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:06,674 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T12:05:06,675 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C38979%2C1733832305620.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:06,689 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620/ef751fafe6b1%2C38979%2C1733832305620.meta.1733832306675.meta 2024-12-10T12:05:06,690 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:06,691 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:06,691 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T12:05:06,691 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T12:05:06,691 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T12:05:06,691 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T12:05:06,691 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:06,691 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T12:05:06,692 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T12:05:06,692 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T12:05:06,693 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T12:05:06,694 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,694 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T12:05:06,695 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T12:05:06,695 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,695 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T12:05:06,696 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T12:05:06,696 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,696 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,696 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T12:05:06,697 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T12:05:06,697 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,698 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:06,698 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T12:05:06,698 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:06,699 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:06,699 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T12:05:06,699 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T12:05:06,699 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:05:06,700 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T12:05:06,700 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71147207, jitterRate=0.060175999999046326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:05:06,700 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T12:05:06,701 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733832306692Writing region info on filesystem at 1733832306692Initializing all the Stores at 1733832306692Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832306692Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832306692Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832306692Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832306692Cleaning up temporary data from old regions at 1733832306699 (+7 ms)Running coprocessor post-open hooks at 1733832306700 (+1 ms)Region opened successfully at 1733832306701 (+1 ms) 2024-12-10T12:05:06,702 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733832306669 2024-12-10T12:05:06,704 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T12:05:06,704 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T12:05:06,704 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,705 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef751fafe6b1,38979,1733832305620, state=OPEN 2024-12-10T12:05:06,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:06,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:06,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:06,735 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:06,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:06,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:06,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:06,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-10T12:05:06,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef751fafe6b1,38979,1733832305620 in 218 msec 2024-12-10T12:05:06,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-10T12:05:06,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 704 msec 2024-12-10T12:05:06,739 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-10T12:05:06,739 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-10T12:05:06,742 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T12:05:06,742 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,38979,1733832305620, seqNum=-1] 2024-12-10T12:05:06,742 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:05:06,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49957, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:05:06,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 778 msec 2024-12-10T12:05:06,750 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733832306749, completionTime=-1 2024-12-10T12:05:06,750 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-12-10T12:05:06,750 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-10T12:05:06,753 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-12-10T12:05:06,753 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733832366753 2024-12-10T12:05:06,753 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733832426753 2024-12-10T12:05:06,753 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 3 msec 2024-12-10T12:05:06,753 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-10T12:05:06,754 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,39461,1733832305455-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,754 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,39461,1733832305455-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,754 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,39461,1733832305455-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,754 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef751fafe6b1:39461, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,754 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,756 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,757 DEBUG [master/ef751fafe6b1:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.067sec 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,39461,1733832305455-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:05:06,761 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,39461,1733832305455-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-10T12:05:06,762 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-10T12:05:06,762 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-10T12:05:06,762 INFO [master/ef751fafe6b1:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,39461,1733832305455-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:06,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:05:06,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef751fafe6b1,39461,-1 for getting cluster id 2024-12-10T12:05:06,856 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-10T12:05:06,858 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a4555d43-9006-4544-83d6-816eee74586d' 2024-12-10T12:05:06,859 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-10T12:05:06,859 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a4555d43-9006-4544-83d6-816eee74586d" 2024-12-10T12:05:06,859 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@384b4593, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:05:06,859 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef751fafe6b1,39461,-1] 2024-12-10T12:05:06,859 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-10T12:05:06,860 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:06,862 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-10T12:05:06,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-10T12:05:06,864 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T12:05:06,865 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,38979,1733832305620, seqNum=-1] 2024-12-10T12:05:06,866 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:05:06,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:05:06,871 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:06,871 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-10T12:05:06,872 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:06,872 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@77510027 2024-12-10T12:05:06,872 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-10T12:05:06,875 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-10T12:05:06,876 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1733832306871', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-10T12:05:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 2024-12-10T12:05:06,877 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 execute state=CREATE_TABLE_PRE_OPERATION 2024-12-10T12:05:06,878 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:06,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1733832306871" procId is: 4 2024-12-10T12:05:06,879 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-10T12:05:06,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:06,889 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 796edff106c225e877c68e562a88b0fa, NAME => 'testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1733832306871', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:06,896 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:06,896 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(1722): Closing 796edff106c225e877c68e562a88b0fa, disabling compactions & flushes 2024-12-10T12:05:06,896 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:06,896 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:06,896 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. after waiting 0 ms 2024-12-10T12:05:06,896 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:06,896 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:06,896 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1733832306871-pool-0 {}] regionserver.HRegion(1676): Region close journal for 796edff106c225e877c68e562a88b0fa: Waiting for close lock at 1733832306896Disabling compacts and flushes for region at 1733832306896Disabling writes for close at 1733832306896Writing region close event to WAL at 1733832306896Closed at 1733832306896 2024-12-10T12:05:06,897 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 execute state=CREATE_TABLE_ADD_TO_META 2024-12-10T12:05:06,898 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1733832306897"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733832306897"}]},"ts":"1733832306897"} 2024-12-10T12:05:06,901 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-10T12:05:06,902 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-10T12:05:06,902 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733832306871","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733832306902"}]},"ts":"1733832306902"} 2024-12-10T12:05:06,905 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733832306871, state=ENABLING in hbase:meta 2024-12-10T12:05:06,905 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {ef751fafe6b1=0} racks are {/default-rack=0} 2024-12-10T12:05:06,906 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T12:05:06,906 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T12:05:06,906 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T12:05:06,906 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T12:05:06,906 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T12:05:06,906 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T12:05:06,906 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T12:05:06,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN}] 2024-12-10T12:05:06,908 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN 2024-12-10T12:05:06,909 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN; state=OFFLINE, location=ef751fafe6b1,38979,1733832305620; forceNewPlan=false, retain=false 2024-12-10T12:05:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:07,059 INFO [ef751fafe6b1:39461 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-10T12:05:07,060 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=796edff106c225e877c68e562a88b0fa, regionState=OPENING, regionLocation=ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:07,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN because future has completed 2024-12-10T12:05:07,064 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796edff106c225e877c68e562a88b0fa, server=ef751fafe6b1,38979,1733832305620}] 2024-12-10T12:05:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:07,218 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:07,218 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 796edff106c225e877c68e562a88b0fa, NAME => 'testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa.', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:07,219 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1733832306871 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,219 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:07,219 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,219 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,220 INFO [StoreOpener-796edff106c225e877c68e562a88b0fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,223 INFO [StoreOpener-796edff106c225e877c68e562a88b0fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 796edff106c225e877c68e562a88b0fa columnFamilyName family 2024-12-10T12:05:07,223 DEBUG [StoreOpener-796edff106c225e877c68e562a88b0fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:07,223 INFO [StoreOpener-796edff106c225e877c68e562a88b0fa-1 {}] regionserver.HStore(327): Store=796edff106c225e877c68e562a88b0fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-10T12:05:07,224 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,224 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,225 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,225 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,225 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,226 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,231 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-10T12:05:07,231 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 796edff106c225e877c68e562a88b0fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70066586, jitterRate=0.04407349228858948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-10T12:05:07,231 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:07,232 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 796edff106c225e877c68e562a88b0fa: Running coprocessor pre-open hook at 1733832307219Writing region info on filesystem at 1733832307219Initializing all the Stores at 1733832307220 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832307220Cleaning up temporary data from old regions at 1733832307225 (+5 ms)Running coprocessor post-open hooks at 1733832307231 (+6 ms)Region opened successfully at 1733832307232 (+1 ms) 2024-12-10T12:05:07,233 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa., pid=6, masterSystemTime=1733832307216 2024-12-10T12:05:07,236 DEBUG [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:07,236 INFO [RS_OPEN_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:07,236 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=796edff106c225e877c68e562a88b0fa, regionState=OPEN, openSeqNum=2, regionLocation=ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:07,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796edff106c225e877c68e562a88b0fa, server=ef751fafe6b1,38979,1733832305620 because future has completed 2024-12-10T12:05:07,242 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-10T12:05:07,243 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 796edff106c225e877c68e562a88b0fa, server=ef751fafe6b1,38979,1733832305620 in 177 msec 2024-12-10T12:05:07,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-10T12:05:07,244 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN in 336 msec 2024-12-10T12:05:07,245 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-10T12:05:07,245 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1733832306871","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733832307245"}]},"ts":"1733832307245"} 2024-12-10T12:05:07,247 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1733832306871, state=ENABLED in hbase:meta 2024-12-10T12:05:07,248 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 execute state=CREATE_TABLE_POST_OPERATION 2024-12-10T12:05:07,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1733832306871 in 372 msec 2024-12-10T12:05:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39461 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-10T12:05:07,508 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1733832306871 completed 2024-12-10T12:05:07,508 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1733832306871 get assigned. Timeout = 60000ms 2024-12-10T12:05:07,508 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T12:05:07,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1733832306871 assigned to meta. Checking AM states. 2024-12-10T12:05:07,512 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-10T12:05:07,512 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1733832306871 assigned. 2024-12-10T12:05:07,516 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1733832306871', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa., hostname=ef751fafe6b1,38979,1733832305620, seqNum=2] 2024-12-10T12:05:07,522 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 796edff106c225e877c68e562a88b0fa 1/1 column families, dataSize=78 B heapSize=480 B 2024-12-10T12:05:07,530 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 9251158516f74126853c252fc178d882 2024-12-10T12:05:07,542 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/.tmp/family/9251158516f74126853c252fc178d882 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-12-10T12:05:07,542 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/.tmp/family/9251158516f74126853c252fc178d882 2024-12-10T12:05:07,545 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/.tmp/family/9251158516f74126853c252fc178d882 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882 2024-12-10T12:05:07,546 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882, delay=995 ms 2024-12-10T12:05:07,547 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882, entries=2, sequenceid=6, filesize=4.9 K 2024-12-10T12:05:07,548 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 796edff106c225e877c68e562a88b0fa in 26ms, sequenceid=6, compaction requested=false 2024-12-10T12:05:07,548 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1733832306871' 2024-12-10T12:05:07,549 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 796edff106c225e877c68e562a88b0fa: 2024-12-10T12:05:07,628 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-10T12:05:08,542 DEBUG [hfile-prefetch-1733832304794 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9251158516f74126853c252fc178d882_0. 2024-12-10T12:05:08,543 DEBUG [hfile-prefetch-1733832304794 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 9251158516f74126853c252fc178d882_117. 2024-12-10T12:05:08,543 DEBUG [hfile-prefetch-1733832304794 {}] bucket.BucketCache(2281): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882, with total blocks 2, and data blocks 1 2024-12-10T12:05:08,543 DEBUG [hfile-prefetch-1733832304794 {}] bucket.BucketCache(2285): iterating over 0 entries in the backing map 2024-12-10T12:05:08,543 DEBUG [hfile-prefetch-1733832304794 {}] bucket.BucketCache(2308): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-12-10T12:05:08,543 DEBUG [hfile-prefetch-1733832304794 {}] bucket.BucketCache(2318): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882. Will sleep 100ms and try the verification again. 2024-12-10T12:05:08,549 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;ef751fafe6b1:38979,5,FailOnTimeoutGroup] 2024-12-10T12:05:08,549 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef751fafe6b1,38979,1733832305620' ***** 2024-12-10T12:05:08,549 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-12-10T12:05:08,549 INFO [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T12:05:08,552 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T12:05:08,553 INFO [regionserver/ef751fafe6b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:08,615 INFO [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence 2024-12-10T12:05:08,615 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T12:05:08,615 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T12:05:08,615 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T12:05:08,615 INFO [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T12:05:08,615 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T12:05:08,625 DEBUG [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1487): Thread RS:1;ef751fafe6b1:38979 started persisting bucket cache to file 2024-12-10T12:05:08,625 DEBUG [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:08,625 DEBUG [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:08,628 DEBUG [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-12-10T12:05:08,628 DEBUG [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1733832308625 2024-12-10T12:05:08,628 DEBUG [RS:1;ef751fafe6b1:38979 {}] bucket.BucketCache(1509): Thread RS:1;ef751fafe6b1:38979 finished persisting bucket cache to file, renaming 2024-12-10T12:05:08,628 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T12:05:08,628 INFO [RS:1;ef751fafe6b1:38979 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T12:05:08,628 INFO [RS:1;ef751fafe6b1:38979 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T12:05:08,628 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(3091): Received CLOSE for 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:08,629 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(959): stopping server ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:08,629 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T12:05:08,629 INFO [RS:1;ef751fafe6b1:38979 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;ef751fafe6b1:38979. 2024-12-10T12:05:08,629 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T12:05:08,629 DEBUG [RS:1;ef751fafe6b1:38979 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:08,629 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 796edff106c225e877c68e562a88b0fa, disabling compactions & flushes 2024-12-10T12:05:08,629 DEBUG [RS:1;ef751fafe6b1:38979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:08,629 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:08,629 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:08,629 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T12:05:08,629 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. after waiting 0 ms 2024-12-10T12:05:08,629 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T12:05:08,629 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:08,629 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T12:05:08,629 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T12:05:08,630 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-10T12:05:08,630 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1325): Online Regions={796edff106c225e877c68e562a88b0fa=testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa., 1588230740=hbase:meta,,1.1588230740} 2024-12-10T12:05:08,630 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T12:05:08,630 DEBUG [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 796edff106c225e877c68e562a88b0fa 2024-12-10T12:05:08,630 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T12:05:08,630 DEBUG [StoreFileCloser-796edff106c225e877c68e562a88b0fa-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882 2024-12-10T12:05:08,630 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T12:05:08,630 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T12:05:08,630 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T12:05:08,630 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T12:05:08,630 WARN [hfile-prefetch-1733832304794 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/family/9251158516f74126853c252fc178d882, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2337) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2320) ~[classes/:?] ... 9 more 2024-12-10T12:05:08,630 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-12-10T12:05:08,631 DEBUG [hfile-prefetch-1733832304794 {}] hfile.PrefetchExecutor(121): Prefetch completed for 9251158516f74126853c252fc178d882 2024-12-10T12:05:08,634 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/default/testBlockEvictionOnGracefulStop1733832306871/796edff106c225e877c68e562a88b0fa/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-10T12:05:08,634 INFO [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:08,634 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 796edff106c225e877c68e562a88b0fa: Waiting for close lock at 1733832308629Running coprocessor pre-close hooks at 1733832308629Disabling compacts and flushes for region at 1733832308629Disabling writes for close at 1733832308629Writing region close event to WAL at 1733832308630 (+1 ms)Running coprocessor post-close hooks at 1733832308634 (+4 ms)Closed at 1733832308634 2024-12-10T12:05:08,634 DEBUG [RS_CLOSE_REGION-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa. 2024-12-10T12:05:08,638 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 268b98c5be7f4bf79e76d250be29d214 2024-12-10T12:05:08,649 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/268b98c5be7f4bf79e76d250be29d214 is 209, key is testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa./info:regioninfo/1733832307236/Put/seqid=0 2024-12-10T12:05:08,650 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/268b98c5be7f4bf79e76d250be29d214 2024-12-10T12:05:08,657 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on fed713edc45d42dcb5734492a70dbb8a 2024-12-10T12:05:08,668 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/ns/fed713edc45d42dcb5734492a70dbb8a is 43, key is default/ns:d/1733832306745/Put/seqid=0 2024-12-10T12:05:08,669 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/ns/fed713edc45d42dcb5734492a70dbb8a 2024-12-10T12:05:08,676 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on f51fa645a9df435f85cd8a425b90b2fc 2024-12-10T12:05:08,687 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/table/f51fa645a9df435f85cd8a425b90b2fc is 80, key is testBlockEvictionOnGracefulStop1733832306871/table:state/1733832307245/Put/seqid=0 2024-12-10T12:05:08,688 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/table/f51fa645a9df435f85cd8a425b90b2fc 2024-12-10T12:05:08,690 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/268b98c5be7f4bf79e76d250be29d214 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/268b98c5be7f4bf79e76d250be29d214 2024-12-10T12:05:08,692 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/268b98c5be7f4bf79e76d250be29d214, delay=920 ms 2024-12-10T12:05:08,693 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/268b98c5be7f4bf79e76d250be29d214, entries=10, sequenceid=11, filesize=7.0 K 2024-12-10T12:05:08,694 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/ns/fed713edc45d42dcb5734492a70dbb8a as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/fed713edc45d42dcb5734492a70dbb8a 2024-12-10T12:05:08,695 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/fed713edc45d42dcb5734492a70dbb8a, delay=954 ms 2024-12-10T12:05:08,696 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/fed713edc45d42dcb5734492a70dbb8a, entries=2, sequenceid=11, filesize=5.0 K 2024-12-10T12:05:08,697 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/table/f51fa645a9df435f85cd8a425b90b2fc as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/f51fa645a9df435f85cd8a425b90b2fc 2024-12-10T12:05:08,698 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/f51fa645a9df435f85cd8a425b90b2fc, delay=914 ms 2024-12-10T12:05:08,699 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/f51fa645a9df435f85cd8a425b90b2fc, entries=2, sequenceid=11, filesize=5.3 K 2024-12-10T12:05:08,699 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-12-10T12:05:08,699 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1487): Thread bucket-cache-persister started persisting bucket cache to file 2024-12-10T12:05:08,699 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:08,699 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:08,699 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=11, compaction requested=false 2024-12-10T12:05:08,700 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/268b98c5be7f4bf79e76d250be29d214 2024-12-10T12:05:08,701 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/fed713edc45d42dcb5734492a70dbb8a 2024-12-10T12:05:08,701 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/f51fa645a9df435f85cd8a425b90b2fc 2024-12-10T12:05:08,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:08,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832308699 2024-12-10T12:05:08,702 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1509): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-12-10T12:05:08,705 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-10T12:05:08,705 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T12:05:08,705 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:08,705 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733832308630Running coprocessor pre-close hooks at 1733832308630Disabling compacts and flushes for region at 1733832308630Disabling writes for close at 1733832308630Obtaining lock to block concurrent updates at 1733832308630Preparing flush snapshotting stores in 1588230740 at 1733832308630Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1733832308631 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733832308631Flushing 1588230740/info: creating writer at 1733832308631Flushing 1588230740/info: appending metadata at 1733832308648 (+17 ms)Flushing 1588230740/info: closing flushed file at 1733832308648Flushing 1588230740/ns: creating writer at 1733832308652 (+4 ms)Flushing 1588230740/ns: appending metadata at 1733832308668 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733832308668Flushing 1588230740/table: creating writer at 1733832308671 (+3 ms)Flushing 1588230740/table: appending metadata at 1733832308687 (+16 ms)Flushing 1588230740/table: closing flushed file at 1733832308687Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31c61d68: reopening flushed file at 1733832308690 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61e220e7: reopening flushed file at 1733832308694 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@30bfccdc: reopening flushed file at 1733832308696 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 69ms, sequenceid=11, compaction requested=false at 1733832308700 (+4 ms)Writing region close event to WAL at 1733832308702 (+2 ms)Running coprocessor post-close hooks at 1733832308705 (+3 ms)Closed at 1733832308705 2024-12-10T12:05:08,705 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:08,830 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(976): stopping server ef751fafe6b1,38979,1733832305620; all regions closed. 2024-12-10T12:05:08,832 DEBUG [RS:1;ef751fafe6b1:38979 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:08,832 INFO [RS:1;ef751fafe6b1:38979 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C38979%2C1733832305620.meta:.meta(num 1733832306675) 2024-12-10T12:05:08,834 DEBUG [RS:1;ef751fafe6b1:38979 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:08,834 INFO [RS:1;ef751fafe6b1:38979 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C38979%2C1733832305620:(num 1733832306283) 2024-12-10T12:05:08,834 DEBUG [RS:1;ef751fafe6b1:38979 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:08,834 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:08,834 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T12:05:08,834 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.ChoreService(370): Chore service for: regionserver/ef751fafe6b1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T12:05:08,834 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T12:05:08,834 INFO [regionserver/ef751fafe6b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T12:05:08,834 INFO [RS:1;ef751fafe6b1:38979 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38979 2024-12-10T12:05:08,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:08,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef751fafe6b1,38979,1733832305620 2024-12-10T12:05:08,876 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T12:05:08,885 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef751fafe6b1,38979,1733832305620] 2024-12-10T12:05:08,893 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef751fafe6b1,38979,1733832305620 already deleted, retry=false 2024-12-10T12:05:08,893 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of ef751fafe6b1,38979,1733832305620 on ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:08,897 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure ef751fafe6b1,38979,1733832305620, splitWal=true, meta=true 2024-12-10T12:05:08,899 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure ef751fafe6b1,38979,1733832305620, splitWal=true, meta=true 2024-12-10T12:05:08,899 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for ef751fafe6b1,38979,1733832305620 (carryingMeta=true) ef751fafe6b1,38979,1733832305620/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@13a86641[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-10T12:05:08,900 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure ef751fafe6b1,38979,1733832305620, splitWal=true, meta=true, isMeta: true 2024-12-10T12:05:08,900 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620-splitting 2024-12-10T12:05:08,901 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620-splitting dir is empty, no logs to split. 2024-12-10T12:05:08,901 INFO [PEWorker-2 {}] master.SplitWALManager(105): ef751fafe6b1,38979,1733832305620 WAL count=0, meta=true 2024-12-10T12:05:08,902 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620-splitting dir is empty, no logs to split. 2024-12-10T12:05:08,902 INFO [PEWorker-2 {}] master.SplitWALManager(105): ef751fafe6b1,38979,1733832305620 WAL count=0, meta=true 2024-12-10T12:05:08,902 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if ef751fafe6b1,38979,1733832305620 WAL splitting is done? wals=0, meta=true 2024-12-10T12:05:08,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-10T12:05:08,905 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T12:05:08,905 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T12:05:08,985 INFO [RS:1;ef751fafe6b1:38979 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T12:05:08,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:08,985 INFO [RS:1;ef751fafe6b1:38979 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef751fafe6b1,38979,1733832305620; zookeeper connection closed. 2024-12-10T12:05:08,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38979-0x1000fab29bd0002, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:08,985 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4b284a08 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4b284a08 2024-12-10T12:05:09,056 DEBUG [ef751fafe6b1:39461 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-10T12:05:09,056 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:09,057 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef751fafe6b1,41493,1733832305580, state=OPENING 2024-12-10T12:05:09,064 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef751fafe6b1:0 server-side Connection retries=45 2024-12-10T12:05:09,065 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:09,065 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:09,065 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-10T12:05:09,065 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-10T12:05:09,065 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-10T12:05:09,065 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-10T12:05:09,065 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-10T12:05:09,066 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46021 2024-12-10T12:05:09,067 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46021 connecting to ZooKeeper ensemble=127.0.0.1:59363 2024-12-10T12:05:09,068 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:09,068 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:09,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:09,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:09,074 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-10T12:05:09,074 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:09,074 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:09,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef751fafe6b1,41493,1733832305580}] 2024-12-10T12:05:09,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460210x0, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-10T12:05:09,085 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-12-10T12:05:09,086 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:460210x0, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-12-10T12:05:09,086 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46021-0x1000fab29bd0003 connected 2024-12-10T12:05:09,087 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-10T12:05:09,087 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.cache 2024-12-10T12:05:09,087 INFO [Time-limited test {}] bucket.BucketCache(364): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-12-10T12:05:09,087 INFO [Time-limited test {}] bucket.BucketCache(478): Starting BucketCachePersisterThread 2024-12-10T12:05:09,087 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-12-10T12:05:09,088 INFO [Thread-208 {}] bucket.BucketCache(1530): Started retrieving bucket cache from file 2024-12-10T12:05:09,088 INFO [Time-limited test {}] bucket.BucketCache(403): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-12-10T12:05:09,088 INFO [Thread-208 {}] bucket.BucketCache(1552): Reading new chunked format of persistence. 2024-12-10T12:05:09,089 INFO [Thread-208 {}] bucket.BucketCache(1705): Retrieved 0 of chunks with blockCount = 0. 2024-12-10T12:05:09,095 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-10T12:05:09,095 DEBUG [Thread-208 {}] bucket.BucketCache(1569): Updating region size map after retrieving cached file list 2024-12-10T12:05:09,096 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-10T12:05:09,096 INFO [Thread-208 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-12-10T12:05:09,096 INFO [Thread-208 {}] bucket.BucketCache(1562): Bucket cache retrieved from file successfully with size: 0 2024-12-10T12:05:09,096 INFO [Thread-208 {}] bucket.BucketCache(413): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence is complete. 2024-12-10T12:05:09,098 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-10T12:05:09,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46021 2024-12-10T12:05:09,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46021 2024-12-10T12:05:09,099 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46021 2024-12-10T12:05:09,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46021 2024-12-10T12:05:09,100 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46021 2024-12-10T12:05:09,101 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(746): ClusterId : a4555d43-9006-4544-83d6-816eee74586d 2024-12-10T12:05:09,102 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-10T12:05:09,111 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-10T12:05:09,111 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-10T12:05:09,119 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-10T12:05:09,120 DEBUG [RS:2;ef751fafe6b1:46021 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5733c2a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef751fafe6b1/172.17.0.2:0 2024-12-10T12:05:09,137 DEBUG [RS:2;ef751fafe6b1:46021 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;ef751fafe6b1:46021 2024-12-10T12:05:09,137 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-10T12:05:09,137 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-10T12:05:09,137 DEBUG [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-10T12:05:09,138 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef751fafe6b1,39461,1733832305455 with port=46021, startcode=1733832309064 2024-12-10T12:05:09,138 DEBUG [RS:2;ef751fafe6b1:46021 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-10T12:05:09,140 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58581, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-12-10T12:05:09,141 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39461 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,141 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39461 {}] master.ServerManager(517): Registering regionserver=ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,143 DEBUG [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b 2024-12-10T12:05:09,143 DEBUG [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-12-10T12:05:09,143 DEBUG [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-10T12:05:09,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:09,152 DEBUG [RS:2;ef751fafe6b1:46021 {}] zookeeper.ZKUtil(111): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,152 WARN [RS:2;ef751fafe6b1:46021 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-10T12:05:09,152 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef751fafe6b1,46021,1733832309064] 2024-12-10T12:05:09,152 INFO [RS:2;ef751fafe6b1:46021 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:09,152 DEBUG [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,158 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-10T12:05:09,161 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-10T12:05:09,161 INFO [RS:2;ef751fafe6b1:46021 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-10T12:05:09,161 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,162 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-10T12:05:09,162 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-10T12:05:09,162 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,162 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef751fafe6b1:0, corePoolSize=2, maxPoolSize=2 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef751fafe6b1:0, corePoolSize=1, maxPoolSize=1 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:09,163 DEBUG [RS:2;ef751fafe6b1:46021 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef751fafe6b1:0, corePoolSize=3, maxPoolSize=3 2024-12-10T12:05:09,165 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,165 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,165 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,165 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,165 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,165 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,46021,1733832309064-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-10T12:05:09,178 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-10T12:05:09,178 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef751fafe6b1,46021,1733832309064-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,179 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,179 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.Replication(171): ef751fafe6b1,46021,1733832309064 started 2024-12-10T12:05:09,192 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-10T12:05:09,192 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(1482): Serving as ef751fafe6b1,46021,1733832309064, RpcServer on ef751fafe6b1/172.17.0.2:46021, sessionid=0x1000fab29bd0003 2024-12-10T12:05:09,192 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-10T12:05:09,192 DEBUG [RS:2;ef751fafe6b1:46021 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,192 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,46021,1733832309064' 2024-12-10T12:05:09,192 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-10T12:05:09,193 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-10T12:05:09,193 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-10T12:05:09,193 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-10T12:05:09,193 DEBUG [RS:2;ef751fafe6b1:46021 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,193 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef751fafe6b1,46021,1733832309064' 2024-12-10T12:05:09,193 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-10T12:05:09,194 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-10T12:05:09,194 DEBUG [RS:2;ef751fafe6b1:46021 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-10T12:05:09,194 INFO [RS:2;ef751fafe6b1:46021 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-10T12:05:09,194 INFO [RS:2;ef751fafe6b1:46021 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-10T12:05:09,227 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-10T12:05:09,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37543, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-10T12:05:09,231 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-10T12:05:09,231 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-10T12:05:09,232 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-10T12:05:09,233 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C41493%2C1733832305580.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41493,1733832305580, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:09,251 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,41493,1733832305580/ef751fafe6b1%2C41493%2C1733832305580.meta.1733832309234.meta 2024-12-10T12:05:09,252 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:09,252 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-10T12:05:09,252 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-10T12:05:09,252 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-10T12:05:09,252 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-10T12:05:09,252 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-10T12:05:09,252 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-10T12:05:09,252 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-10T12:05:09,253 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-10T12:05:09,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-12-10T12:05:09,254 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-10T12:05:09,254 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:09,255 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/268b98c5be7f4bf79e76d250be29d214, delay=952 ms 2024-12-10T12:05:09,257 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/268b98c5be7f4bf79e76d250be29d214 2024-12-10T12:05:09,257 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:09,257 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-12-10T12:05:09,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-10T12:05:09,258 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:09,260 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/fed713edc45d42dcb5734492a70dbb8a, delay=907 ms 2024-12-10T12:05:09,261 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/fed713edc45d42dcb5734492a70dbb8a 2024-12-10T12:05:09,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:09,261 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-12-10T12:05:09,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-10T12:05:09,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:09,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:09,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-12-10T12:05:09,262 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-10T12:05:09,263 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-10T12:05:09,264 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/f51fa645a9df435f85cd8a425b90b2fc, delay=996 ms 2024-12-10T12:05:09,265 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/f51fa645a9df435f85cd8a425b90b2fc 2024-12-10T12:05:09,265 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-10T12:05:09,266 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-10T12:05:09,266 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:09,267 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740 2024-12-10T12:05:09,267 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-10T12:05:09,267 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-10T12:05:09,267 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-10T12:05:09,268 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-10T12:05:09,268 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64980542, jitterRate=-0.03171446919441223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-10T12:05:09,268 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-10T12:05:09,269 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733832309253Writing region info on filesystem at 1733832309253Initializing all the Stores at 1733832309253Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832309253Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832309253Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733832309253Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733832309253Cleaning up temporary data from old regions at 1733832309267 (+14 ms)Running coprocessor post-open hooks at 1733832309268 (+1 ms)Region opened successfully at 1733832309269 (+1 ms) 2024-12-10T12:05:09,270 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1733832309227 2024-12-10T12:05:09,273 DEBUG [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-10T12:05:09,273 INFO [RS_OPEN_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-10T12:05:09,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:09,274 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef751fafe6b1,41493,1733832305580, state=OPEN 2024-12-10T12:05:09,278 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39461 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=ef751fafe6b1,41493,1733832305580, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-10T12:05:09,295 INFO [RS:2;ef751fafe6b1:46021 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-10T12:05:09,296 INFO [RS:2;ef751fafe6b1:46021 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=ef751fafe6b1%2C46021%2C1733832309064, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,46021,1733832309064, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs, maxLogs=55 2024-12-10T12:05:09,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:09,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:09,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-10T12:05:09,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:09,301 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:09,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:09,301 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-10T12:05:09,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-12-10T12:05:09,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef751fafe6b1,41493,1733832305580 in 227 msec 2024-12-10T12:05:09,306 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-10T12:05:09,307 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 400 msec 2024-12-10T12:05:09,307 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): ef751fafe6b1,38979,1733832305620 had 2 regions 2024-12-10T12:05:09,308 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure ef751fafe6b1,38979,1733832305620, splitWal=true, meta=true, isMeta: false 2024-12-10T12:05:09,309 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620-splitting dir is empty, no logs to split. 2024-12-10T12:05:09,309 INFO [PEWorker-3 {}] master.SplitWALManager(105): ef751fafe6b1,38979,1733832305620 WAL count=0, meta=false 2024-12-10T12:05:09,310 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,38979,1733832305620-splitting dir is empty, no logs to split. 2024-12-10T12:05:09,310 INFO [PEWorker-3 {}] master.SplitWALManager(105): ef751fafe6b1,38979,1733832305620 WAL count=0, meta=false 2024-12-10T12:05:09,310 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if ef751fafe6b1,38979,1733832305620 WAL splitting is done? wals=0, meta=false 2024-12-10T12:05:09,312 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN}] 2024-12-10T12:05:09,313 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN 2024-12-10T12:05:09,314 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-10T12:05:09,317 INFO [RS:2;ef751fafe6b1:46021 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/WALs/ef751fafe6b1,46021,1733832309064/ef751fafe6b1%2C46021%2C1733832309064.1733832309296 2024-12-10T12:05:09,317 DEBUG [RS:2;ef751fafe6b1:46021 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-12-10T12:05:09,465 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(204): Hosts are {ef751fafe6b1=0} racks are {/default-rack=0} 2024-12-10T12:05:09,465 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-10T12:05:09,465 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-10T12:05:09,465 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-10T12:05:09,465 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-10T12:05:09,465 INFO [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-10T12:05:09,465 INFO [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-10T12:05:09,465 DEBUG [ef751fafe6b1:39461 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-10T12:05:09,465 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=796edff106c225e877c68e562a88b0fa, regionState=OPENING, regionLocation=ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:09,467 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server ef751fafe6b1:38979 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ef751fafe6b1/172.17.0.2:38979 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:05:09,468 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,38979,1733832305620, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,38979,1733832305620, seqNum=-1, error=java.net.ConnectException: Call to address=ef751fafe6b1:38979 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ef751fafe6b1/172.17.0.2:38979 2024-12-10T12:05:09,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,38979,1733832305620, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-10T12:05:09,469 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,38979,1733832305620, seqNum=-1 from cache 2024-12-10T12:05:09,470 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address ef751fafe6b1:38979 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: ef751fafe6b1/172.17.0.2:38979 2024-12-10T12:05:09,576 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-10T12:05:09,576 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef751fafe6b1,41493,1733832305580, seqNum=-1] 2024-12-10T12:05:09,577 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-10T12:05:09,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33653, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-10T12:05:09,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa, ASSIGN because future has completed 2024-12-10T12:05:09,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796edff106c225e877c68e562a88b0fa, server=ef751fafe6b1,41493,1733832305580}] 2024-12-10T12:05:09,616 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-10T12:05:09,616 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-10T12:05:09,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T12:05:09,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-10T12:05:09,617 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-10T12:05:09,617 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-10T12:05:09,692 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-10T12:05:09,692 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T12:05:09,692 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:09,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:09,693 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:09,693 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-10T12:05:09,693 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-10T12:05:09,693 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1818987156, stopped=false 2024-12-10T12:05:09,693 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef751fafe6b1,39461,1733832305455 2024-12-10T12:05:09,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:09,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:09,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-10T12:05:09,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:09,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:09,718 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:09,718 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T12:05:09,718 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-10T12:05:09,718 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:09,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:09,719 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:09,719 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef751fafe6b1,41493,1733832305580' ***** 2024-12-10T12:05:09,719 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T12:05:09,719 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef751fafe6b1,46021,1733832309064' ***** 2024-12-10T12:05:09,719 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-10T12:05:09,719 INFO [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T12:05:09,719 INFO [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1738): Disabling cache 2024-12-10T12:05:09,719 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:09,719 INFO [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence 2024-12-10T12:05:09,719 INFO [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1762): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/de45c626-f0b8-15cf-a0c5-518b1799032e/bucket.persistence 2024-12-10T12:05:09,719 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T12:05:09,719 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T12:05:09,719 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-12-10T12:05:09,720 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-12-10T12:05:09,720 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T12:05:09,720 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1271): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-12-10T12:05:09,720 INFO [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T12:05:09,720 INFO [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1768): Shutting down cache persister thread. 2024-12-10T12:05:09,720 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-10T12:05:09,720 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T12:05:09,720 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-12-10T12:05:09,730 DEBUG [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1487): Thread RS:0;ef751fafe6b1:41493 started persisting bucket cache to file 2024-12-10T12:05:09,730 DEBUG [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1487): Thread RS:2;ef751fafe6b1:46021 started persisting bucket cache to file 2024-12-10T12:05:09,730 DEBUG [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:09,730 DEBUG [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1494): Persist in new chunked persistence format. 2024-12-10T12:05:09,730 DEBUG [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:09,730 DEBUG [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1673): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-12-10T12:05:09,733 DEBUG [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:09,733 DEBUG [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832309730 2024-12-10T12:05:09,733 ERROR [RS:0;ef751fafe6b1:41493 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1678) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1496) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1774) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-10T12:05:09,734 DEBUG [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1680): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-12-10T12:05:09,734 DEBUG [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1498): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1733832309730 2024-12-10T12:05:09,734 DEBUG [RS:0;ef751fafe6b1:41493 {}] bucket.BucketCache(1509): Thread RS:0;ef751fafe6b1:41493 finished persisting bucket cache to file, renaming 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T12:05:09,734 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(959): stopping server ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef751fafe6b1:41493. 2024-12-10T12:05:09,734 DEBUG [RS:0;ef751fafe6b1:41493 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:09,734 DEBUG [RS:0;ef751fafe6b1:41493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T12:05:09,734 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T12:05:09,735 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T12:05:09,735 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-10T12:05:09,735 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-10T12:05:09,735 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-10T12:05:09,733 DEBUG [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(271): Request to ef751fafe6b1,41493,1733832305580 failed, try=0 org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ef751fafe6b1:41493 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-12-10T12:05:09,735 DEBUG [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-10T12:05:09,735 WARN [RSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): ef751fafe6b1,41493,1733832305580 tells us DoNotRetry due to org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ef751fafe6b1:41493 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException, try=0, give up 2024-12-10T12:05:09,735 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-12-10T12:05:09,735 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-10T12:05:09,735 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-10T12:05:09,735 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-10T12:05:09,735 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-10T12:05:09,735 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-10T12:05:09,735 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=485 B heapSize=1.70 KB 2024-12-10T12:05:09,735 WARN [RSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(139): The remote operation pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 796edff106c225e877c68e562a88b0fa, server=ef751fafe6b1,41493,1733832305580 for region state=OPENING, location=ef751fafe6b1,41493,1733832305580, table=testBlockEvictionOnGracefulStop1733832306871, region=796edff106c225e877c68e562a88b0fa to server ef751fafe6b1,41493,1733832305580 failed org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=ef751fafe6b1:41493 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos$AdminService$Stub.executeProcedures(AdminProtos.java:35059) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.lambda$executeProcedures$20(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:103) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.call(AsyncRegionServerAdmin.java:121) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRegionServerAdmin.executeProcedures(AsyncRegionServerAdmin.java:218) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.sendRequest(RSProcedureDispatcher.java:431) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:398) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 11 more 2024-12-10T12:05:09,744 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 7693d4965ae54cc5975a3e487bd2010d 2024-12-10T12:05:09,750 DEBUG [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1509): Thread RS:2;ef751fafe6b1:46021 finished persisting bucket cache to file, renaming 2024-12-10T12:05:09,750 WARN [RS:2;ef751fafe6b1:46021 {}] bucket.BucketCache(1512): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-12-10T12:05:09,750 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-10T12:05:09,751 INFO [RS:2;ef751fafe6b1:46021 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-10T12:05:09,751 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-10T12:05:09,751 INFO [RS:2;ef751fafe6b1:46021 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-10T12:05:09,751 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(959): stopping server ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,751 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T12:05:09,751 INFO [RS:2;ef751fafe6b1:46021 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;ef751fafe6b1:46021. 2024-12-10T12:05:09,751 DEBUG [RS:2;ef751fafe6b1:46021 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-10T12:05:09,751 DEBUG [RS:2;ef751fafe6b1:46021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:09,751 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(976): stopping server ef751fafe6b1,46021,1733832309064; all regions closed. 2024-12-10T12:05:09,753 DEBUG [RS:2;ef751fafe6b1:46021 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:09,753 INFO [RS:2;ef751fafe6b1:46021 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C46021%2C1733832309064:(num 1733832309296) 2024-12-10T12:05:09,753 DEBUG [RS:2;ef751fafe6b1:46021 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:09,753 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:09,753 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T12:05:09,754 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.ChoreService(370): Chore service for: regionserver/ef751fafe6b1:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-10T12:05:09,754 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-10T12:05:09,754 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-10T12:05:09,754 INFO [regionserver/ef751fafe6b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T12:05:09,754 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-10T12:05:09,754 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T12:05:09,754 INFO [RS:2;ef751fafe6b1:46021 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46021 2024-12-10T12:05:09,754 INFO [regionserver/ef751fafe6b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:09,758 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/7693d4965ae54cc5975a3e487bd2010d is 209, key is testBlockEvictionOnGracefulStop1733832306871,,1733832306875.796edff106c225e877c68e562a88b0fa./info:regioninfo/1733832309465/Put/seqid=0 2024-12-10T12:05:09,759 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=485 B at sequenceid=18 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/7693d4965ae54cc5975a3e487bd2010d 2024-12-10T12:05:09,761 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/.tmp/info/7693d4965ae54cc5975a3e487bd2010d as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/7693d4965ae54cc5975a3e487bd2010d 2024-12-10T12:05:09,762 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/7693d4965ae54cc5975a3e487bd2010d, delay=902 ms 2024-12-10T12:05:09,763 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/7693d4965ae54cc5975a3e487bd2010d, entries=3, sequenceid=18, filesize=5.9 K 2024-12-10T12:05:09,764 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=18, compaction requested=false 2024-12-10T12:05:09,764 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/268b98c5be7f4bf79e76d250be29d214 2024-12-10T12:05:09,765 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/info/7693d4965ae54cc5975a3e487bd2010d 2024-12-10T12:05:09,765 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/ns/fed713edc45d42dcb5734492a70dbb8a 2024-12-10T12:05:09,766 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/table/f51fa645a9df435f85cd8a425b90b2fc 2024-12-10T12:05:09,766 INFO [regionserver/ef751fafe6b1:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:09,769 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/data/hbase/meta/1588230740/recovered.edits/21.seqid, newMaxSeqId=21, maxSeqId=14 2024-12-10T12:05:09,770 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-10T12:05:09,770 INFO [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:09,770 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733832309735Running coprocessor pre-close hooks at 1733832309735Disabling compacts and flushes for region at 1733832309735Disabling writes for close at 1733832309735Obtaining lock to block concurrent updates at 1733832309736 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1733832309736Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=485, getHeapSize=1672, getOffHeapSize=0, getCellsCount=3 at 1733832309736Flushing stores of hbase:meta,,1.1588230740 at 1733832309736Flushing 1588230740/info: creating writer at 1733832309736Flushing 1588230740/info: appending metadata at 1733832309758 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733832309758Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58b56042: reopening flushed file at 1733832309761 (+3 ms)Finished flush of dataSize ~485 B/485, heapSize ~952 B/952, currentSize=0 B/0 for 1588230740 in 29ms, sequenceid=18, compaction requested=false at 1733832309764 (+3 ms)Writing region close event to WAL at 1733832309766 (+2 ms)Running coprocessor post-close hooks at 1733832309770 (+4 ms)Closed at 1733832309770 2024-12-10T12:05:09,770 DEBUG [RS_CLOSE_META-regionserver/ef751fafe6b1:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-10T12:05:09,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:09,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef751fafe6b1,46021,1733832309064 2024-12-10T12:05:09,776 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T12:05:09,790 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef751fafe6b1,46021,1733832309064] 2024-12-10T12:05:09,801 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef751fafe6b1,46021,1733832309064 already deleted, retry=false 2024-12-10T12:05:09,801 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef751fafe6b1,46021,1733832309064 expired; onlineServers=1 2024-12-10T12:05:09,891 INFO [RS:2;ef751fafe6b1:46021 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T12:05:09,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:09,891 INFO [RS:2;ef751fafe6b1:46021 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef751fafe6b1,46021,1733832309064; zookeeper connection closed. 2024-12-10T12:05:09,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46021-0x1000fab29bd0003, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:09,891 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6392e59e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6392e59e 2024-12-10T12:05:09,935 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(976): stopping server ef751fafe6b1,41493,1733832305580; all regions closed. 2024-12-10T12:05:09,937 DEBUG [RS:0;ef751fafe6b1:41493 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:09,937 INFO [RS:0;ef751fafe6b1:41493 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C41493%2C1733832305580.meta:.meta(num 1733832309234) 2024-12-10T12:05:09,938 DEBUG [RS:0;ef751fafe6b1:41493 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/oldWALs 2024-12-10T12:05:09,938 INFO [RS:0;ef751fafe6b1:41493 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef751fafe6b1%2C41493%2C1733832305580:(num 1733832306294) 2024-12-10T12:05:09,939 DEBUG [RS:0;ef751fafe6b1:41493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-10T12:05:09,939 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.LeaseManager(133): Closed leases 2024-12-10T12:05:09,939 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T12:05:09,939 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.ChoreService(370): Chore service for: regionserver/ef751fafe6b1:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-10T12:05:09,939 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T12:05:09,939 INFO [regionserver/ef751fafe6b1:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T12:05:09,939 INFO [RS:0;ef751fafe6b1:41493 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41493 2024-12-10T12:05:09,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-10T12:05:09,974 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef751fafe6b1,41493,1733832305580 2024-12-10T12:05:09,974 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T12:05:09,984 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef751fafe6b1,41493,1733832305580] 2024-12-10T12:05:09,992 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef751fafe6b1,41493,1733832305580 already deleted, retry=false 2024-12-10T12:05:09,993 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef751fafe6b1,41493,1733832305580 expired; onlineServers=0 2024-12-10T12:05:09,993 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef751fafe6b1,39461,1733832305455' ***** 2024-12-10T12:05:09,993 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-10T12:05:09,993 INFO [M:0;ef751fafe6b1:39461 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-10T12:05:09,993 INFO [M:0;ef751fafe6b1:39461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-10T12:05:09,993 DEBUG [M:0;ef751fafe6b1:39461 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-10T12:05:09,993 DEBUG [M:0;ef751fafe6b1:39461 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-10T12:05:09,993 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-10T12:05:09,993 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.small.0-1733832305978 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.small.0-1733832305978,5,FailOnTimeoutGroup] 2024-12-10T12:05:09,993 DEBUG [master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.large.0-1733832305978 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef751fafe6b1:0:becomeActiveMaster-HFileCleaner.large.0-1733832305978,5,FailOnTimeoutGroup] 2024-12-10T12:05:09,993 INFO [M:0;ef751fafe6b1:39461 {}] hbase.ChoreService(370): Chore service for: master/ef751fafe6b1:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-10T12:05:09,993 INFO [M:0;ef751fafe6b1:39461 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-10T12:05:09,993 DEBUG [M:0;ef751fafe6b1:39461 {}] master.HMaster(1795): Stopping service threads 2024-12-10T12:05:09,993 INFO [M:0;ef751fafe6b1:39461 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-10T12:05:09,993 INFO [M:0;ef751fafe6b1:39461 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-10T12:05:09,994 INFO [M:0;ef751fafe6b1:39461 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-10T12:05:09,994 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-10T12:05:10,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-10T12:05:10,001 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-10T12:05:10,001 DEBUG [M:0;ef751fafe6b1:39461 {}] zookeeper.ZKUtil(347): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-10T12:05:10,001 WARN [M:0;ef751fafe6b1:39461 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-10T12:05:10,001 INFO [M:0;ef751fafe6b1:39461 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/.lastflushedseqids 2024-12-10T12:05:10,007 INFO [M:0;ef751fafe6b1:39461 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-10T12:05:10,007 INFO [M:0;ef751fafe6b1:39461 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-10T12:05:10,007 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-10T12:05:10,007 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:10,007 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:10,007 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-10T12:05:10,007 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:10,007 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.93 KB heapSize=44.14 KB 2024-12-10T12:05:10,015 DEBUG [M:0;ef751fafe6b1:39461 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 825eaa247c3c40a58ab998087b0f5245 2024-12-10T12:05:10,028 DEBUG [M:0;ef751fafe6b1:39461 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/825eaa247c3c40a58ab998087b0f5245 is 82, key is hbase:meta,,1/info:regioninfo/1733832309273/Put/seqid=0 2024-12-10T12:05:10,028 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/825eaa247c3c40a58ab998087b0f5245 2024-12-10T12:05:10,035 DEBUG [M:0;ef751fafe6b1:39461 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8081ea179fde427299888ffa1b0eb680 2024-12-10T12:05:10,048 DEBUG [M:0;ef751fafe6b1:39461 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8081ea179fde427299888ffa1b0eb680 is 781, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733832307249/Put/seqid=0 2024-12-10T12:05:10,049 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.70 KB at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8081ea179fde427299888ffa1b0eb680 2024-12-10T12:05:10,057 DEBUG [M:0;ef751fafe6b1:39461 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1d848acc227f4f7fae352a929fd5720e 2024-12-10T12:05:10,069 DEBUG [M:0;ef751fafe6b1:39461 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1d848acc227f4f7fae352a929fd5720e is 69, key is ef751fafe6b1,41493,1733832305580/rs:state/1733832306101/Put/seqid=0 2024-12-10T12:05:10,070 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=90 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1d848acc227f4f7fae352a929fd5720e 2024-12-10T12:05:10,072 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d848acc227f4f7fae352a929fd5720e 2024-12-10T12:05:10,072 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/825eaa247c3c40a58ab998087b0f5245 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/825eaa247c3c40a58ab998087b0f5245 2024-12-10T12:05:10,074 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/825eaa247c3c40a58ab998087b0f5245, entries=14, sequenceid=90, filesize=5.9 K 2024-12-10T12:05:10,074 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8081ea179fde427299888ffa1b0eb680 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8081ea179fde427299888ffa1b0eb680 2024-12-10T12:05:10,076 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8081ea179fde427299888ffa1b0eb680, entries=11, sequenceid=90, filesize=7.8 K 2024-12-10T12:05:10,076 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1d848acc227f4f7fae352a929fd5720e as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1d848acc227f4f7fae352a929fd5720e 2024-12-10T12:05:10,078 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 1d848acc227f4f7fae352a929fd5720e 2024-12-10T12:05:10,078 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/766af6b8-2ce9-6077-d07f-d671e389639c/e72604e2-503c-f16e-b2c4-fa9a4885924b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1d848acc227f4f7fae352a929fd5720e, entries=3, sequenceid=90, filesize=5.3 K 2024-12-10T12:05:10,079 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.93 KB/35768, heapSize ~43.84 KB/44896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=90, compaction requested=false 2024-12-10T12:05:10,080 INFO [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-10T12:05:10,080 DEBUG [M:0;ef751fafe6b1:39461 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733832310007Disabling compacts and flushes for region at 1733832310007Disabling writes for close at 1733832310007Obtaining lock to block concurrent updates at 1733832310007Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733832310007Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35768, getHeapSize=45136, getOffHeapSize=0, getCellsCount=111 at 1733832310008 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733832310008Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733832310008Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733832310027 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733832310027Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733832310030 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733832310048 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733832310048Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733832310051 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733832310068 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733832310068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79382c5: reopening flushed file at 1733832310072 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a0c0476: reopening flushed file at 1733832310074 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14bf61cf: reopening flushed file at 1733832310076 (+2 ms)Finished flush of dataSize ~34.93 KB/35768, heapSize ~43.84 KB/44896, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 71ms, sequenceid=90, compaction requested=false at 1733832310079 (+3 ms)Writing region close event to WAL at 1733832310080 (+1 ms)Closed at 1733832310080 2024-12-10T12:05:10,082 INFO [M:0;ef751fafe6b1:39461 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-10T12:05:10,082 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-10T12:05:10,082 INFO [M:0;ef751fafe6b1:39461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39461 2024-12-10T12:05:10,082 INFO [M:0;ef751fafe6b1:39461 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-10T12:05:10,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:10,084 INFO [RS:0;ef751fafe6b1:41493 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T12:05:10,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41493-0x1000fab29bd0001, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:10,084 INFO [RS:0;ef751fafe6b1:41493 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef751fafe6b1,41493,1733832305580; zookeeper connection closed. 2024-12-10T12:05:10,085 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b1b553d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b1b553d 2024-12-10T12:05:10,085 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-10T12:05:10,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:10,199 INFO [M:0;ef751fafe6b1:39461 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-10T12:05:10,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39461-0x1000fab29bd0000, quorum=127.0.0.1:59363, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-10T12:05:10,201 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-10T12:05:10,202 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-12-10T12:05:10,202 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-10T12:05:10,208 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=70 (was 49) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=141 (was 141), ProcessCount=11 (was 11), AvailableMemoryMB=6458 (was 6541)