2024-11-08 13:35:30,209 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 13:35:30,231 main DEBUG Took 0.019381 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 13:35:30,232 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 13:35:30,232 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 13:35:30,234 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 13:35:30,235 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,246 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 13:35:30,262 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,264 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,265 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,265 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,266 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,267 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,268 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,268 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,269 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,270 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,271 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,271 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,272 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,273 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,273 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,274 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,274 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,276 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,277 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,277 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,278 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 13:35:30,279 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,279 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 13:35:30,282 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 13:35:30,284 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 13:35:30,286 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 13:35:30,287 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 13:35:30,289 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 13:35:30,290 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 13:35:30,301 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 13:35:30,304 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 13:35:30,307 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 13:35:30,308 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 13:35:30,308 main DEBUG createAppenders(={Console}) 2024-11-08 13:35:30,309 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 initialized 2024-11-08 13:35:30,310 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 2024-11-08 13:35:30,310 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@15a34df2 OK. 2024-11-08 13:35:30,311 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 13:35:30,311 main DEBUG OutputStream closed 2024-11-08 13:35:30,312 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 13:35:30,312 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 13:35:30,313 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4f2b503c OK 2024-11-08 13:35:30,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 13:35:30,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 13:35:30,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 13:35:30,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 13:35:30,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 13:35:30,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 13:35:30,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 13:35:30,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 13:35:30,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 13:35:30,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 13:35:30,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 13:35:30,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 13:35:30,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 13:35:30,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 13:35:30,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 13:35:30,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 13:35:30,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 13:35:30,426 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 13:35:30,428 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 13:35:30,428 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@31c7528f) with optional ClassLoader: null 2024-11-08 13:35:30,429 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 13:35:30,429 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@31c7528f] started OK. 2024-11-08T13:35:30,441 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement timeout: 13 mins 2024-11-08 13:35:30,443 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 13:35:30,444 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T13:35:30,733 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950 2024-11-08T13:35:30,762 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=185, ProcessCount=11, AvailableMemoryMB=3740 2024-11-08T13:35:30,872 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T13:35:30,978 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:30,995 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/cluster_79f6a00b-f8ff-b1f2-0da7-0b842f284658, deleteOnExit=true 2024-11-08T13:35:31,097 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/cluster_79f6a00b-f8ff-b1f2-0da7-0b842f284658/zookeeper_0, clientPort=65101, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/cluster_79f6a00b-f8ff-b1f2-0da7-0b842f284658/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/cluster_79f6a00b-f8ff-b1f2-0da7-0b842f284658/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T13:35:31,111 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=65101 2024-11-08T13:35:31,112 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:31,197 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a with version=8 2024-11-08T13:35:31,198 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/hbase-staging 2024-11-08T13:35:31,289 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T13:35:31,540 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:35:31,550 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:31,551 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:31,559 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:35:31,559 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:31,560 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:35:31,690 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T13:35:31,769 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:35:31,803 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T13:35:31,813 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T13:35:31,816 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:35:31,852 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 42771 (auto-detected) 2024-11-08T13:35:31,856 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T13:35:31,879 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45857 2024-11-08T13:35:31,908 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45857 connecting to ZooKeeper ensemble=127.0.0.1:65101 2024-11-08T13:35:31,942 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:458570x0, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:35:31,945 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45857-0x1001f8c116e0000 connected 2024-11-08T13:35:31,976 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:31,977 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:31,987 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:31,991 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a, hbase.cluster.distributed=false 2024-11-08T13:35:32,019 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:35:32,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45857 2024-11-08T13:35:32,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45857 2024-11-08T13:35:32,026 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45857 2024-11-08T13:35:32,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45857 2024-11-08T13:35:32,029 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45857 2024-11-08T13:35:32,112 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:35:32,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:32,114 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:32,114 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:35:32,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:32,115 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:35:32,117 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:35:32,120 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:35:32,121 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42175 2024-11-08T13:35:32,123 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42175 connecting to ZooKeeper ensemble=127.0.0.1:65101 2024-11-08T13:35:32,124 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:32,125 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:32,128 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421750x0, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:35:32,129 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421750x0, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:32,129 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42175-0x1001f8c116e0001 connected 2024-11-08T13:35:32,135 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:35:32,148 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.cache 2024-11-08T13:35:32,156 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T13:35:32,157 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T13:35:32,159 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T13:35:32,161 INFO [Thread-10 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T13:35:32,162 WARN [Thread-10 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T13:35:32,169 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T13:35:32,173 INFO [Thread-10 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T13:35:32,173 INFO [Thread-10 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.persistence is complete. 2024-11-08T13:35:32,175 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:35:32,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:35:32,185 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:35:32,186 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42175 2024-11-08T13:35:32,186 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42175 2024-11-08T13:35:32,187 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42175 2024-11-08T13:35:32,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42175 2024-11-08T13:35:32,188 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42175 2024-11-08T13:35:32,214 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:35:32,215 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:32,215 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:32,215 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:35:32,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:32,216 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:35:32,216 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:35:32,217 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:35:32,218 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35971 2024-11-08T13:35:32,220 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35971 connecting to ZooKeeper ensemble=127.0.0.1:65101 2024-11-08T13:35:32,221 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:32,222 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:32,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359710x0, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:35:32,228 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359710x0, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:32,228 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35971-0x1001f8c116e0002 connected 2024-11-08T13:35:32,229 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:35:32,234 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.cache 2024-11-08T13:35:32,234 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T13:35:32,234 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T13:35:32,235 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T13:35:32,235 INFO [Thread-11 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T13:35:32,235 WARN [Thread-11 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T13:35:32,236 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T13:35:32,236 INFO [Thread-11 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T13:35:32,236 INFO [Thread-11 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.persistence is complete. 2024-11-08T13:35:32,237 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:35:32,238 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:35:32,241 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:35:32,244 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35971 2024-11-08T13:35:32,245 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35971 2024-11-08T13:35:32,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35971 2024-11-08T13:35:32,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35971 2024-11-08T13:35:32,253 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35971 2024-11-08T13:35:32,271 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6ccfea20c990:45857 2024-11-08T13:35:32,273 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6ccfea20c990,45857,1731072931375 2024-11-08T13:35:32,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:32,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:32,280 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:32,282 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6ccfea20c990,45857,1731072931375 2024-11-08T13:35:32,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:35:32,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:35:32,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:32,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:32,311 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:32,312 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T13:35:32,314 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6ccfea20c990,45857,1731072931375 from backup master directory 2024-11-08T13:35:32,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:32,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:32,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6ccfea20c990,45857,1731072931375 2024-11-08T13:35:32,318 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:32,319 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:35:32,319 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6ccfea20c990,45857,1731072931375 2024-11-08T13:35:32,322 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T13:35:32,324 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T13:35:32,337 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/hbase.id] with ID: 0265b489-7c34-4cc2-8b51-61459e8de875 2024-11-08T13:35:32,337 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.tmp/hbase.id 2024-11-08T13:35:32,344 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/hbase.id] 2024-11-08T13:35:32,389 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:32,395 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T13:35:32,415 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 18ms. 2024-11-08T13:35:32,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:32,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:32,419 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:32,449 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:35:32,452 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T13:35:32,495 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:35:32,502 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:32,557 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store 2024-11-08T13:35:32,576 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T13:35:32,580 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:32,581 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:35:32,581 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:32,582 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:32,583 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:35:32,583 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:32,584 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:32,584 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731072932581Disabling compacts and flushes for region at 1731072932581Disabling writes for close at 1731072932583 (+2 ms)Writing region close event to WAL at 1731072932583Closed at 1731072932583 2024-11-08T13:35:32,585 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/.initializing 2024-11-08T13:35:32,585 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/WALs/6ccfea20c990,45857,1731072931375 2024-11-08T13:35:32,596 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:35:32,607 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C45857%2C1731072931375, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/WALs/6ccfea20c990,45857,1731072931375, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/oldWALs, maxLogs=10 2024-11-08T13:35:32,642 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/WALs/6ccfea20c990,45857,1731072931375/6ccfea20c990%2C45857%2C1731072931375.1731072932611 2024-11-08T13:35:32,642 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:32,643 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:32,643 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:32,647 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,648 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T13:35:32,735 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:32,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:32,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,743 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T13:35:32,744 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:32,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:32,745 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T13:35:32,750 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:32,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:32,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,756 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T13:35:32,756 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:32,757 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:32,757 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,762 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,763 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,767 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,768 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,772 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:35:32,776 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:32,782 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:35:32,783 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74745129, jitterRate=0.11378921568393707}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:35:32,793 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731072932660Initializing all the Stores at 1731072932661 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072932661Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072932662 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072932663 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072932663Cleaning up temporary data from old regions at 1731072932769 (+106 ms)Region opened successfully at 1731072932793 (+24 ms) 2024-11-08T13:35:32,795 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T13:35:32,833 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b69bdf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:35:32,874 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T13:35:32,890 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T13:35:32,890 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T13:35:32,894 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:35:32,895 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T13:35:32,902 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-08T13:35:32,902 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T13:35:32,936 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T13:35:32,949 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T13:35:32,952 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T13:35:32,956 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T13:35:32,958 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T13:35:32,960 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T13:35:32,963 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T13:35:32,967 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T13:35:32,969 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T13:35:32,972 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T13:35:32,973 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T13:35:32,992 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T13:35:32,993 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T13:35:33,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:33,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:33,005 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:33,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,010 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6ccfea20c990,45857,1731072931375, sessionid=0x1001f8c116e0000, setting cluster-up flag (Was=false) 2024-11-08T13:35:33,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,025 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,032 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T13:35:33,035 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,45857,1731072931375 2024-11-08T13:35:33,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,045 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T13:35:33,048 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,45857,1731072931375 2024-11-08T13:35:33,053 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T13:35:33,058 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(746): ClusterId : 0265b489-7c34-4cc2-8b51-61459e8de875 2024-11-08T13:35:33,058 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(746): ClusterId : 0265b489-7c34-4cc2-8b51-61459e8de875 2024-11-08T13:35:33,060 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:35:33,060 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:35:33,065 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:35:33,065 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:35:33,066 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:35:33,066 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:35:33,069 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:35:33,069 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:35:33,070 DEBUG [RS:0;6ccfea20c990:42175 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e280438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:35:33,070 DEBUG [RS:1;6ccfea20c990:35971 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@211591c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:35:33,092 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6ccfea20c990:42175 2024-11-08T13:35:33,093 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6ccfea20c990:35971 2024-11-08T13:35:33,096 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:35:33,096 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:35:33,096 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:35:33,096 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:35:33,096 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:35:33,096 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:35:33,099 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,45857,1731072931375 with port=42175, startcode=1731072932076 2024-11-08T13:35:33,099 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,45857,1731072931375 with port=35971, startcode=1731072932214 2024-11-08T13:35:33,112 DEBUG [RS:1;6ccfea20c990:35971 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:35:33,112 DEBUG [RS:0;6ccfea20c990:42175 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:35:33,140 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:33,151 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T13:35:33,166 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T13:35:33,173 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6ccfea20c990,45857,1731072931375 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T13:35:33,184 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:33,185 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:33,185 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:33,185 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:33,185 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6ccfea20c990:0, corePoolSize=10, maxPoolSize=10 2024-11-08T13:35:33,186 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,186 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:35:33,187 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,187 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39725, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:35:33,187 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37667, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:35:33,190 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072963190 2024-11-08T13:35:33,192 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T13:35:33,193 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:33,194 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T13:35:33,194 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T13:35:33,195 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:35:33,198 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T13:35:33,198 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T13:35:33,199 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T13:35:33,199 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T13:35:33,199 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,200 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T13:35:33,202 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T13:35:33,202 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,207 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T13:35:33,208 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T13:35:33,209 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T13:35:33,212 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T13:35:33,212 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T13:35:33,217 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072933214,5,FailOnTimeoutGroup] 2024-11-08T13:35:33,221 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072933217,5,FailOnTimeoutGroup] 2024-11-08T13:35:33,222 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,222 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T13:35:33,223 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,224 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,225 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T13:35:33,225 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T13:35:33,225 WARN [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T13:35:33,226 WARN [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T13:35:33,237 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T13:35:33,238 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:33,251 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:33,253 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T13:35:33,261 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:35:33,262 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,262 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,263 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T13:35:33,267 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:35:33,267 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,268 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T13:35:33,273 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:35:33,273 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,273 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,274 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T13:35:33,278 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:35:33,278 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,279 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,279 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:35:33,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:33,281 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:33,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:35:33,282 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:35:33,282 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:35:33,283 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:35:33,290 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:35:33,291 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70551749, jitterRate=0.05130298435688019}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:35:33,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731072933251Initializing all the Stores at 1731072933252 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072933252Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072933253 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072933253Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072933253Cleaning up temporary data from old regions at 1731072933282 (+29 ms)Region opened successfully at 1731072933296 (+14 ms) 2024-11-08T13:35:33,297 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:35:33,297 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:35:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:35:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:35:33,298 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:35:33,300 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:33,300 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731072933297Disabling compacts and flushes for region at 1731072933297Disabling writes for close at 1731072933298 (+1 ms)Writing region close event to WAL at 1731072933299 (+1 ms)Closed at 1731072933300 (+1 ms) 2024-11-08T13:35:33,304 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:33,304 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T13:35:33,315 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T13:35:33,326 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:35:33,327 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,45857,1731072931375 with port=35971, startcode=1731072932214 2024-11-08T13:35:33,327 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,45857,1731072931375 with port=42175, startcode=1731072932076 2024-11-08T13:35:33,329 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T13:35:33,330 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,35971,1731072932214 2024-11-08T13:35:33,333 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,35971,1731072932214 2024-11-08T13:35:33,343 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,42175,1731072932076 2024-11-08T13:35:33,343 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45857 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,42175,1731072932076 2024-11-08T13:35:33,343 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:33,343 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T13:35:33,343 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:35:33,345 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:33,346 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T13:35:33,346 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:35:33,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:33,351 DEBUG [RS:0;6ccfea20c990:42175 {}] zookeeper.ZKUtil(111): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,42175,1731072932076 2024-11-08T13:35:33,351 WARN [RS:0;6ccfea20c990:42175 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:35:33,351 INFO [RS:0;6ccfea20c990:42175 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:33,351 DEBUG [RS:1;6ccfea20c990:35971 {}] zookeeper.ZKUtil(111): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,35971,1731072932214 2024-11-08T13:35:33,352 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,42175,1731072932076 2024-11-08T13:35:33,352 WARN [RS:1;6ccfea20c990:35971 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:35:33,352 INFO [RS:1;6ccfea20c990:35971 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:33,352 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,35971,1731072932214 2024-11-08T13:35:33,354 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,42175,1731072932076] 2024-11-08T13:35:33,355 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,35971,1731072932214] 2024-11-08T13:35:33,383 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:35:33,383 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:35:33,402 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:35:33,402 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:35:33,407 INFO [RS:1;6ccfea20c990:35971 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:35:33,407 INFO [RS:0;6ccfea20c990:42175 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:35:33,407 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,407 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,408 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:35:33,409 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:35:33,415 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:35:33,415 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:35:33,417 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,417 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,417 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,418 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:35:33,418 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:35:33,419 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,419 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,420 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,420 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:33,420 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:33,420 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:33,420 DEBUG [RS:0;6ccfea20c990:42175 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:33,420 DEBUG [RS:1;6ccfea20c990:35971 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:33,425 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35971,1731072932214-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,426 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,427 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,42175,1731072932076-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:35:33,452 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:35:33,452 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:35:33,455 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,42175,1731072932076-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,455 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,35971,1731072932214-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,455 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,455 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,455 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.Replication(171): 6ccfea20c990,42175,1731072932076 started 2024-11-08T13:35:33,456 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.Replication(171): 6ccfea20c990,35971,1731072932214 started 2024-11-08T13:35:33,478 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,478 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,42175,1731072932076, RpcServer on 6ccfea20c990/172.17.0.2:42175, sessionid=0x1001f8c116e0001 2024-11-08T13:35:33,480 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:35:33,480 DEBUG [RS:0;6ccfea20c990:42175 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,42175,1731072932076 2024-11-08T13:35:33,480 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,42175,1731072932076' 2024-11-08T13:35:33,480 WARN [6ccfea20c990:45857 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:35:33,480 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:35:33,481 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:35:33,482 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:33,482 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:35:33,482 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,35971,1731072932214, RpcServer on 6ccfea20c990/172.17.0.2:35971, sessionid=0x1001f8c116e0002 2024-11-08T13:35:33,482 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:35:33,482 DEBUG [RS:0;6ccfea20c990:42175 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,42175,1731072932076 2024-11-08T13:35:33,482 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,42175,1731072932076' 2024-11-08T13:35:33,483 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:35:33,483 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:35:33,483 DEBUG [RS:1;6ccfea20c990:35971 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,35971,1731072932214 2024-11-08T13:35:33,483 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35971,1731072932214' 2024-11-08T13:35:33,483 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:35:33,483 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:35:33,484 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:35:33,484 DEBUG [RS:0;6ccfea20c990:42175 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:35:33,484 INFO [RS:0;6ccfea20c990:42175 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:35:33,484 INFO [RS:0;6ccfea20c990:42175 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:35:33,484 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:35:33,484 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:35:33,484 DEBUG [RS:1;6ccfea20c990:35971 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,35971,1731072932214 2024-11-08T13:35:33,484 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,35971,1731072932214' 2024-11-08T13:35:33,485 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:35:33,489 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:35:33,490 DEBUG [RS:1;6ccfea20c990:35971 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:35:33,490 INFO [RS:1;6ccfea20c990:35971 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:35:33,491 INFO [RS:1;6ccfea20c990:35971 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:35:33,589 INFO [RS:0;6ccfea20c990:42175 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:35:33,590 INFO [RS:0;6ccfea20c990:42175 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C42175%2C1731072932076, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,42175,1731072932076, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:33,592 INFO [RS:1;6ccfea20c990:35971 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:35:33,593 INFO [RS:1;6ccfea20c990:35971 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C35971%2C1731072932214, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,35971,1731072932214, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:33,610 INFO [RS:0;6ccfea20c990:42175 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,42175,1731072932076/6ccfea20c990%2C42175%2C1731072932076.1731072933593 2024-11-08T13:35:33,610 INFO [RS:1;6ccfea20c990:35971 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,35971,1731072932214/6ccfea20c990%2C35971%2C1731072932214.1731072933595 2024-11-08T13:35:33,611 DEBUG [RS:0;6ccfea20c990:42175 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:33,611 DEBUG [RS:1;6ccfea20c990:35971 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:33,732 DEBUG [6ccfea20c990:45857 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T13:35:33,741 DEBUG [6ccfea20c990:45857 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:35:33,744 DEBUG [6ccfea20c990:45857 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:35:33,744 DEBUG [6ccfea20c990:45857 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:35:33,744 DEBUG [6ccfea20c990:45857 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:35:33,744 DEBUG [6ccfea20c990:45857 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:35:33,744 INFO [6ccfea20c990:45857 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:35:33,744 INFO [6ccfea20c990:45857 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:35:33,744 DEBUG [6ccfea20c990:45857 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:35:33,750 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ccfea20c990,35971,1731072932214 2024-11-08T13:35:33,755 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35971,1731072932214, state=OPENING 2024-11-08T13:35:33,759 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T13:35:33,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,760 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:33,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:33,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:33,761 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:33,763 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:35:33,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35971,1731072932214}] 2024-11-08T13:35:33,936 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:35:33,939 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41327, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:35:33,947 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T13:35:33,947 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:33,948 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T13:35:33,949 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C35971%2C1731072932214.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,35971,1731072932214, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:33,969 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,35971,1731072932214/6ccfea20c990%2C35971%2C1731072932214.meta.1731072933951.meta 2024-11-08T13:35:33,970 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:33,970 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:33,972 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T13:35:33,975 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T13:35:33,980 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T13:35:33,984 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T13:35:33,985 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:33,985 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T13:35:33,985 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T13:35:33,987 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T13:35:33,988 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:35:33,988 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,989 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T13:35:33,990 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:35:33,990 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,990 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T13:35:33,991 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:35:33,991 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,991 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T13:35:33,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:35:33,992 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:33,993 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:33,993 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:35:33,994 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:33,995 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:33,995 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:35:33,995 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:35:33,995 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:35:33,997 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:35:33,997 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66403826, jitterRate=-0.010505884885787964}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:35:33,998 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T13:35:34,000 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731072933986Writing region info on filesystem at 1731072933986Initializing all the Stores at 1731072933987 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072933987Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072933987Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072933987Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072933987Cleaning up temporary data from old regions at 1731072933995 (+8 ms)Running coprocessor post-open hooks at 1731072933998 (+3 ms)Region opened successfully at 1731072934000 (+2 ms) 2024-11-08T13:35:34,005 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731072933928 2024-11-08T13:35:34,015 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T13:35:34,016 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T13:35:34,017 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35971,1731072932214 2024-11-08T13:35:34,018 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,35971,1731072932214, state=OPEN 2024-11-08T13:35:34,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:34,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:34,020 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:34,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:34,020 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:34,021 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:34,021 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ccfea20c990,35971,1731072932214 2024-11-08T13:35:34,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T13:35:34,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,35971,1731072932214 in 257 msec 2024-11-08T13:35:34,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T13:35:34,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 712 msec 2024-11-08T13:35:34,029 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:34,029 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T13:35:34,050 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:35:34,051 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35971,1731072932214, seqNum=-1] 2024-11-08T13:35:34,069 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:35:34,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58911, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:35:34,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 992 msec 2024-11-08T13:35:34,087 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731072934087, completionTime=-1 2024-11-08T13:35:34,090 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T13:35:34,090 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:35:34,113 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:35:34,113 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072994113 2024-11-08T13:35:34,113 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731073054113 2024-11-08T13:35:34,113 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-08T13:35:34,115 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-08T13:35:34,121 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,45857,1731072931375-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:34,121 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,45857,1731072931375-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:34,121 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,45857,1731072931375-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:34,122 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6ccfea20c990:45857, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:34,123 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:34,123 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:34,129 DEBUG [master/6ccfea20c990:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T13:35:34,150 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.830sec 2024-11-08T13:35:34,151 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T13:35:34,152 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T13:35:34,153 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T13:35:34,154 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T13:35:34,154 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T13:35:34,155 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,45857,1731072931375-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:35:34,155 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,45857,1731072931375-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T13:35:34,156 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T13:35:34,157 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T13:35:34,157 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,45857,1731072931375-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:34,166 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4248d59d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:35:34,168 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T13:35:34,168 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T13:35:34,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6ccfea20c990,45857,-1 for getting cluster id 2024-11-08T13:35:34,174 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T13:35:34,182 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0265b489-7c34-4cc2-8b51-61459e8de875' 2024-11-08T13:35:34,184 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T13:35:34,185 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0265b489-7c34-4cc2-8b51-61459e8de875" 2024-11-08T13:35:34,187 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca13f8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:35:34,187 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6ccfea20c990,45857,-1] 2024-11-08T13:35:34,190 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T13:35:34,191 DEBUG [RPCClient-NioEventLoopGroup-5-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:34,193 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T13:35:34,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f59369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:35:34,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:35:34,203 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,35971,1731072932214, seqNum=-1] 2024-11-08T13:35:34,204 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:35:34,206 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:35:34,237 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6ccfea20c990,45857,1731072931375 2024-11-08T13:35:34,242 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T13:35:34,250 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 6ccfea20c990,45857,1731072931375 2024-11-08T13:35:34,256 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@777312b7 2024-11-08T13:35:34,258 DEBUG [RPCClient-NioEventLoopGroup-5-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T13:35:34,262 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T13:35:34,268 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnRegionMove1731072934237', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:35:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 2024-11-08T13:35:34,276 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:35:34,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnRegionMove1731072934237" procId is: 4 2024-11-08T13:35:34,278 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:34,280 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:35:34,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:34,295 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 79ff66faf3951b7dd82433c4237b679d, NAME => 'testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnRegionMove1731072934237', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:34,302 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:34,303 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(1722): Closing 79ff66faf3951b7dd82433c4237b679d, disabling compactions & flushes 2024-11-08T13:35:34,303 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:34,303 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:34,303 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. after waiting 0 ms 2024-11-08T13:35:34,303 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:34,303 INFO [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:34,303 DEBUG [RegionOpenAndInit-testBlockEvictionOnRegionMove1731072934237-pool-0 {}] regionserver.HRegion(1676): Region close journal for 79ff66faf3951b7dd82433c4237b679d: Waiting for close lock at 1731072934303Disabling compacts and flushes for region at 1731072934303Disabling writes for close at 1731072934303Writing region close event to WAL at 1731072934303Closed at 1731072934303 2024-11-08T13:35:34,305 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:35:34,309 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731072934305"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072934305"}]},"ts":"1731072934305"} 2024-11-08T13:35:34,313 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:35:34,314 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:35:34,316 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731072934237","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731072934314"}]},"ts":"1731072934314"} 2024-11-08T13:35:34,321 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731072934237, state=ENABLING in hbase:meta 2024-11-08T13:35:34,321 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:35:34,322 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:35:34,322 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:35:34,322 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:35:34,322 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:35:34,322 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:35:34,322 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:35:34,322 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:35:34,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, ASSIGN}] 2024-11-08T13:35:34,326 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, ASSIGN 2024-11-08T13:35:34,327 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, ASSIGN; state=OFFLINE, location=6ccfea20c990,35971,1731072932214; forceNewPlan=false, retain=false 2024-11-08T13:35:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:34,479 INFO [6ccfea20c990:45857 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T13:35:34,480 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=79ff66faf3951b7dd82433c4237b679d, regionState=OPENING, regionLocation=6ccfea20c990,35971,1731072932214 2024-11-08T13:35:34,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, ASSIGN because future has completed 2024-11-08T13:35:34,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,35971,1731072932214}] 2024-11-08T13:35:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:34,641 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:34,642 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 79ff66faf3951b7dd82433c4237b679d, NAME => 'testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:34,642 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731072934237 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,643 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:34,643 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,643 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,644 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,647 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79ff66faf3951b7dd82433c4237b679d columnFamilyName family 2024-11-08T13:35:34,648 DEBUG [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:34,648 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.HStore(327): Store=79ff66faf3951b7dd82433c4237b679d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:34,648 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,650 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,650 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,650 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,650 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,651 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,657 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:35:34,658 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 79ff66faf3951b7dd82433c4237b679d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64089539, jitterRate=-0.044991448521614075}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:35:34,658 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:34,659 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 79ff66faf3951b7dd82433c4237b679d: Running coprocessor pre-open hook at 1731072934643Writing region info on filesystem at 1731072934643Initializing all the Stores at 1731072934643Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072934643Cleaning up temporary data from old regions at 1731072934650 (+7 ms)Running coprocessor post-open hooks at 1731072934658 (+8 ms)Region opened successfully at 1731072934659 (+1 ms) 2024-11-08T13:35:34,660 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d., pid=6, masterSystemTime=1731072934638 2024-11-08T13:35:34,663 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:34,663 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:34,663 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=79ff66faf3951b7dd82433c4237b679d, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,35971,1731072932214 2024-11-08T13:35:34,667 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,35971,1731072932214 because future has completed 2024-11-08T13:35:34,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T13:35:34,672 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,35971,1731072932214 in 184 msec 2024-11-08T13:35:34,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T13:35:34,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, ASSIGN in 348 msec 2024-11-08T13:35:34,676 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:35:34,676 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnRegionMove1731072934237","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731072934676"}]},"ts":"1731072934676"} 2024-11-08T13:35:34,680 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnRegionMove1731072934237, state=ENABLED in hbase:meta 2024-11-08T13:35:34,682 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:35:34,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnRegionMove1731072934237 in 412 msec 2024-11-08T13:35:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:34,912 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnRegionMove1731072934237 get assigned. Timeout = 60000ms 2024-11-08T13:35:34,912 INFO [RPCClient-NioEventLoopGroup-5-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnRegionMove1731072934237 completed 2024-11-08T13:35:34,915 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:35:34,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnRegionMove1731072934237 assigned to meta. Checking AM states. 2024-11-08T13:35:34,922 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:35:34,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnRegionMove1731072934237 assigned. 2024-11-08T13:35:34,933 DEBUG [RPCClient-NioEventLoopGroup-5-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnRegionMove1731072934237', row='row1', locateType=CURRENT is [region=testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d., hostname=6ccfea20c990,35971,1731072932214, seqNum=2] 2024-11-08T13:35:34,945 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 79ff66faf3951b7dd82433c4237b679d 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T13:35:34,968 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:35,007 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/.tmp/family/23b61f7ee1934ea9a59da26b2aa6510f is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T13:35:35,013 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/.tmp/family/23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:35,051 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/.tmp/family/23b61f7ee1934ea9a59da26b2aa6510f as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:35,052 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f, delay=983 ms 2024-11-08T13:35:35,054 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T13:35:35,062 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for 79ff66faf3951b7dd82433c4237b679d in 115ms, sequenceid=6, compaction requested=false 2024-11-08T13:35:35,063 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnRegionMove1731072934237' 2024-11-08T13:35:35,066 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 79ff66faf3951b7dd82433c4237b679d: 2024-11-08T13:35:36,038 DEBUG [hfile-prefetch-1731072935052 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 23b61f7ee1934ea9a59da26b2aa6510f_0. 2024-11-08T13:35:36,043 DEBUG [hfile-prefetch-1731072935052 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 23b61f7ee1934ea9a59da26b2aa6510f_117. 2024-11-08T13:35:36,044 DEBUG [hfile-prefetch-1731072935052 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f, with total blocks 2, and data blocks 1 2024-11-08T13:35:36,044 DEBUG [hfile-prefetch-1731072935052 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T13:35:36,044 DEBUG [hfile-prefetch-1731072935052 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T13:35:36,045 DEBUG [hfile-prefetch-1731072935052 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f. Will sleep 100ms and try the verification again. 2024-11-08T13:35:36,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=79ff66faf3951b7dd82433c4237b679d, source=6ccfea20c990,35971,1731072932214, destination=6ccfea20c990,42175,1731072932076, warming up region on 6ccfea20c990,42175,1731072932076 2024-11-08T13:35:36,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:35:36,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=79ff66faf3951b7dd82433c4237b679d, source=6ccfea20c990,35971,1731072932214, destination=6ccfea20c990,42175,1731072932076, running balancer 2024-11-08T13:35:36,081 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56025, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:35:36,082 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.RSRpcServices(2066): Warmup testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(7855): Warmup {ENCODED => 79ff66faf3951b7dd82433c4237b679d, NAME => 'testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:36,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:36,083 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,084 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79ff66faf3951b7dd82433c4237b679d columnFamilyName family 2024-11-08T13:35:36,084 DEBUG [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, REOPEN/MOVE 2024-11-08T13:35:36,119 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, REOPEN/MOVE 2024-11-08T13:35:36,123 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=79ff66faf3951b7dd82433c4237b679d, regionState=CLOSING, regionLocation=6ccfea20c990,35971,1731072932214 2024-11-08T13:35:36,124 DEBUG [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:36,125 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.HStore(327): Store=79ff66faf3951b7dd82433c4237b679d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(1722): Closing 79ff66faf3951b7dd82433c4237b679d, disabling compactions & flushes 2024-11-08T13:35:36,125 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. after waiting 0 ms 2024-11-08T13:35:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,126 DEBUG [StoreFileCloser-79ff66faf3951b7dd82433c4237b679d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:36,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,127 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, REOPEN/MOVE because future has completed 2024-11-08T13:35:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42175 {}] regionserver.HRegion(1676): Region close journal for 79ff66faf3951b7dd82433c4237b679d: Waiting for close lock at 1731072936125Disabling compacts and flushes for region at 1731072936125Disabling writes for close at 1731072936125Writing region close event to WAL at 1731072936127 (+2 ms)Closed at 1731072936127 2024-11-08T13:35:36,128 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: true 2024-11-08T13:35:36,128 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,35971,1731072932214}] 2024-11-08T13:35:36,129 WARN [hfile-prefetch-1731072935052 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-08T13:35:36,130 DEBUG [hfile-prefetch-1731072935052 {}] hfile.PrefetchExecutor(121): Prefetch completed for 23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:36,138 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T13:35:36,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T13:35:36,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:36,138 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:36,158 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T13:35:36,159 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731072936138 2024-11-08T13:35:36,159 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T13:35:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45857 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-08T13:35:36,286 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,286 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: true 2024-11-08T13:35:36,287 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 79ff66faf3951b7dd82433c4237b679d, disabling compactions & flushes 2024-11-08T13:35:36,287 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,287 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,287 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. after waiting 0 ms 2024-11-08T13:35:36,287 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,288 DEBUG [StoreFileCloser-79ff66faf3951b7dd82433c4237b679d-family-1 {}] bucket.BucketCache(1864): found 1 blocks for file 23b61f7ee1934ea9a59da26b2aa6510f, starting offset: 0, end offset: 9223372036854775807 2024-11-08T13:35:36,290 DEBUG [StoreFileCloser-79ff66faf3951b7dd82433c4237b679d-family-1 {}] bucket.BucketCache(857): removed key 23b61f7ee1934ea9a59da26b2aa6510f_0 from back map with offset lock 2094080 in the evict process 2024-11-08T13:35:36,294 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T13:35:36,295 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,296 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 79ff66faf3951b7dd82433c4237b679d: Waiting for close lock at 1731072936287Running coprocessor pre-close hooks at 1731072936287Disabling compacts and flushes for region at 1731072936287Disabling writes for close at 1731072936287Writing region close event to WAL at 1731072936290 (+3 ms)Running coprocessor post-close hooks at 1731072936294 (+4 ms)Closed at 1731072936295 (+1 ms) 2024-11-08T13:35:36,296 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 79ff66faf3951b7dd82433c4237b679d move to 6ccfea20c990,42175,1731072932076 record at close sequenceid=6 2024-11-08T13:35:36,299 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,300 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=79ff66faf3951b7dd82433c4237b679d, regionState=CLOSED 2024-11-08T13:35:36,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,35971,1731072932214 because future has completed 2024-11-08T13:35:36,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T13:35:36,306 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,35971,1731072932214 in 176 msec 2024-11-08T13:35:36,307 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, REOPEN/MOVE; state=CLOSED, location=6ccfea20c990,42175,1731072932076; forceNewPlan=false, retain=false 2024-11-08T13:35:36,359 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T13:35:36,359 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T13:35:36,359 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:36,359 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:36,362 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T13:35:36,362 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731072936359 2024-11-08T13:35:36,362 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T13:35:36,458 INFO [6ccfea20c990:45857 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T13:35:36,458 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=79ff66faf3951b7dd82433c4237b679d, regionState=OPENING, regionLocation=6ccfea20c990,42175,1731072932076 2024-11-08T13:35:36,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, REOPEN/MOVE because future has completed 2024-11-08T13:35:36,462 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,42175,1731072932076}] 2024-11-08T13:35:36,617 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,617 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 79ff66faf3951b7dd82433c4237b679d, NAME => 'testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:36,618 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnRegionMove1731072934237 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,618 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:36,619 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,619 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,621 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,622 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 79ff66faf3951b7dd82433c4237b679d columnFamilyName family 2024-11-08T13:35:36,622 DEBUG [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:36,624 DEBUG [StoreFileOpener-79ff66faf3951b7dd82433c4237b679d-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f, delay=953 ms 2024-11-08T13:35:36,625 DEBUG [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:36,626 INFO [StoreOpener-79ff66faf3951b7dd82433c4237b679d-1 {}] regionserver.HStore(327): Store=79ff66faf3951b7dd82433c4237b679d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:36,626 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,627 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,628 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,628 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,628 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,634 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,635 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 79ff66faf3951b7dd82433c4237b679d; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61807838, jitterRate=-0.07899144291877747}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:35:36,635 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,636 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 79ff66faf3951b7dd82433c4237b679d: Running coprocessor pre-open hook at 1731072936620Writing region info on filesystem at 1731072936620Initializing all the Stores at 1731072936620Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072936620Cleaning up temporary data from old regions at 1731072936628 (+8 ms)Running coprocessor post-open hooks at 1731072936635 (+7 ms)Region opened successfully at 1731072936636 (+1 ms) 2024-11-08T13:35:36,637 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d., pid=9, masterSystemTime=1731072936614 2024-11-08T13:35:36,640 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,640 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=79ff66faf3951b7dd82433c4237b679d, regionState=OPEN, openSeqNum=10, regionLocation=6ccfea20c990,42175,1731072932076 2024-11-08T13:35:36,644 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,42175,1731072932076 because future has completed 2024-11-08T13:35:36,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-08T13:35:36,648 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 79ff66faf3951b7dd82433c4237b679d, server=6ccfea20c990,42175,1731072932076 in 184 msec 2024-11-08T13:35:36,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnRegionMove1731072934237, region=79ff66faf3951b7dd82433c4237b679d, REOPEN/MOVE in 569 msec 2024-11-08T13:35:36,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T13:35:36,673 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:35:36,673 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:36,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:36,679 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:36,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T13:35:36,680 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T13:35:36,680 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1482236467, stopped=false 2024-11-08T13:35:36,680 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6ccfea20c990,45857,1731072931375 2024-11-08T13:35:36,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:36,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:36,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:36,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:36,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:36,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:36,682 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:35:36,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:36,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:36,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:36,683 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:35:36,683 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:36,683 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:36,684 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,42175,1731072932076' ***** 2024-11-08T13:35:36,684 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T13:35:36,684 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,35971,1731072932214' ***** 2024-11-08T13:35:36,684 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T13:35:36,684 INFO [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T13:35:36,684 INFO [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T13:35:36,684 INFO [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.persistence 2024-11-08T13:35:36,684 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T13:35:36,684 INFO [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/bucket.persistence 2024-11-08T13:35:36,684 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T13:35:36,684 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T13:35:36,684 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T13:35:36,685 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T13:35:36,685 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T13:35:36,685 INFO [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T13:35:36,685 INFO [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T13:35:36,685 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T13:35:36,685 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T13:35:36,695 DEBUG [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1488): Thread RS:1;6ccfea20c990:35971 started persisting bucket cache to file 2024-11-08T13:35:36,695 DEBUG [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1488): Thread RS:0;6ccfea20c990:42175 started persisting bucket cache to file 2024-11-08T13:35:36,695 DEBUG [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:36,695 DEBUG [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:36,695 DEBUG [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:36,695 DEBUG [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:36,698 DEBUG [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T13:35:36,698 DEBUG [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731072936695 2024-11-08T13:35:36,698 ERROR [RS:1;6ccfea20c990:35971 {}] bucket.PersistentIOEngine(78): Calculating checksum failed, because of java.io.IOException: Stream closed at java.io.BufferedInputStream.getBufIfOpen(BufferedInputStream.java:168) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:334) ~[?:?] at sun.nio.cs.StreamDecoder.readBytes(StreamDecoder.java:281) ~[?:?] at sun.nio.cs.StreamDecoder.implRead(StreamDecoder.java:324) ~[?:?] at sun.nio.cs.StreamDecoder.read(StreamDecoder.java:189) ~[?:?] at java.io.InputStreamReader.read(InputStreamReader.java:177) ~[?:?] at java.io.BufferedReader.fill(BufferedReader.java:162) ~[?:?] at java.io.BufferedReader.read1(BufferedReader.java:213) ~[?:?] at java.io.BufferedReader.read(BufferedReader.java:287) ~[?:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.parseExecResult(Shell.java:1295) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.runCommand(Shell.java:1054) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell.run(Shell.java:959) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.util.Shell$ShellCommandExecutor.execute(Shell.java:1282) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.getFileSize(PersistentIOEngine.java:94) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.PersistentIOEngine.calculateChecksum(PersistentIOEngine.java:71) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.toPB(BucketProtoUtils.java:61) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketProtoUtils.serializeAsPB(BucketProtoUtils.java:75) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistChunkedBackingMap(BucketCache.java:1679) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.persistToFile(BucketCache.java:1497) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.shutdown(BucketCache.java:1775) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.shutdown(CombinedBlockCache.java:169) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:927) ~[classes/:?] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.security.AccessController.doPrivileged(AccessController.java:399) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:376) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:35:36,699 DEBUG [RS:0;6ccfea20c990:42175 {}] bucket.BucketCache(1510): Thread RS:0;6ccfea20c990:42175 finished persisting bucket cache to file, renaming 2024-11-08T13:35:36,699 DEBUG [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T13:35:36,699 DEBUG [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731072936695 2024-11-08T13:35:36,699 DEBUG [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1510): Thread RS:1;6ccfea20c990:35971 finished persisting bucket cache to file, renaming 2024-11-08T13:35:36,699 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:35:36,699 WARN [RS:1;6ccfea20c990:35971 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-08T13:35:36,699 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:35:36,699 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:35:36,699 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:35:36,699 INFO [RS:0;6ccfea20c990:42175 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:35:36,699 INFO [RS:1;6ccfea20c990:35971 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:35:36,699 INFO [RS:1;6ccfea20c990:35971 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:35:36,699 INFO [RS:0;6ccfea20c990:42175 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:35:36,699 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,35971,1731072932214 2024-11-08T13:35:36,700 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(3091): Received CLOSE for 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,700 INFO [RS:1;6ccfea20c990:35971 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:35:36,700 INFO [RS:1;6ccfea20c990:35971 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6ccfea20c990:35971. 2024-11-08T13:35:36,700 DEBUG [RS:1;6ccfea20c990:35971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:36,700 DEBUG [RS:1;6ccfea20c990:35971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:36,700 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:35:36,700 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:35:36,700 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:35:36,700 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,42175,1731072932076 2024-11-08T13:35:36,700 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T13:35:36,700 INFO [RS:0;6ccfea20c990:42175 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:35:36,700 INFO [RS:0;6ccfea20c990:42175 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6ccfea20c990:42175. 2024-11-08T13:35:36,700 DEBUG [RS:0;6ccfea20c990:42175 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:36,700 DEBUG [RS:0;6ccfea20c990:42175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:36,701 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T13:35:36,701 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T13:35:36,701 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1325): Online Regions={79ff66faf3951b7dd82433c4237b679d=testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d.} 2024-11-08T13:35:36,701 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T13:35:36,701 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T13:35:36,701 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T13:35:36,701 DEBUG [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1351): Waiting on 79ff66faf3951b7dd82433c4237b679d 2024-11-08T13:35:36,701 DEBUG [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T13:35:36,701 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 79ff66faf3951b7dd82433c4237b679d, disabling compactions & flushes 2024-11-08T13:35:36,701 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:35:36,701 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,701 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:35:36,701 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,701 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:35:36,701 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. after waiting 0 ms 2024-11-08T13:35:36,701 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,701 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:35:36,702 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:35:36,702 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=3.70 KB heapSize=6.72 KB 2024-11-08T13:35:36,702 DEBUG [StoreFileCloser-79ff66faf3951b7dd82433c4237b679d-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/family/23b61f7ee1934ea9a59da26b2aa6510f 2024-11-08T13:35:36,707 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnRegionMove1731072934237/79ff66faf3951b7dd82433c4237b679d/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T13:35:36,708 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,708 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 79ff66faf3951b7dd82433c4237b679d: Waiting for close lock at 1731072936700Running coprocessor pre-close hooks at 1731072936701 (+1 ms)Disabling compacts and flushes for region at 1731072936701Disabling writes for close at 1731072936701Writing region close event to WAL at 1731072936703 (+2 ms)Running coprocessor post-close hooks at 1731072936708 (+5 ms)Closed at 1731072936708 2024-11-08T13:35:36,708 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d. 2024-11-08T13:35:36,713 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on d30d28b04b2446378cac660c624f5fdb 2024-11-08T13:35:36,738 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:36,738 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:36,738 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/d30d28b04b2446378cac660c624f5fdb is 205, key is testBlockEvictionOnRegionMove1731072934237,,1731072934263.79ff66faf3951b7dd82433c4237b679d./info:regioninfo/1731072936641/Put/seqid=0 2024-11-08T13:35:36,739 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.48 KB at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/d30d28b04b2446378cac660c624f5fdb 2024-11-08T13:35:36,748 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 3a52b131890947759e9c37c74722adf8 2024-11-08T13:35:36,761 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/ns/3a52b131890947759e9c37c74722adf8 is 43, key is default/ns:d/1731072934075/Put/seqid=0 2024-11-08T13:35:36,761 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/ns/3a52b131890947759e9c37c74722adf8 2024-11-08T13:35:36,770 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6c829f035b7c46ffb8dbfc9dada047f4 2024-11-08T13:35:36,783 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/table/6c829f035b7c46ffb8dbfc9dada047f4 is 78, key is testBlockEvictionOnRegionMove1731072934237/table:state/1731072934676/Put/seqid=0 2024-11-08T13:35:36,783 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=15 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/table/6c829f035b7c46ffb8dbfc9dada047f4 2024-11-08T13:35:36,786 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/d30d28b04b2446378cac660c624f5fdb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/d30d28b04b2446378cac660c624f5fdb 2024-11-08T13:35:36,788 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/d30d28b04b2446378cac660c624f5fdb, delay=973 ms 2024-11-08T13:35:36,789 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/d30d28b04b2446378cac660c624f5fdb, entries=15, sequenceid=15, filesize=7.6 K 2024-11-08T13:35:36,790 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/ns/3a52b131890947759e9c37c74722adf8 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/3a52b131890947759e9c37c74722adf8 2024-11-08T13:35:36,791 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/3a52b131890947759e9c37c74722adf8, delay=902 ms 2024-11-08T13:35:36,793 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/3a52b131890947759e9c37c74722adf8, entries=2, sequenceid=15, filesize=5.0 K 2024-11-08T13:35:36,793 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/table/6c829f035b7c46ffb8dbfc9dada047f4 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/6c829f035b7c46ffb8dbfc9dada047f4 2024-11-08T13:35:36,795 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/6c829f035b7c46ffb8dbfc9dada047f4, delay=923 ms 2024-11-08T13:35:36,796 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/6c829f035b7c46ffb8dbfc9dada047f4, entries=2, sequenceid=15, filesize=5.3 K 2024-11-08T13:35:36,797 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false 2024-11-08T13:35:36,797 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T13:35:36,799 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/d30d28b04b2446378cac660c624f5fdb 2024-11-08T13:35:36,800 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/3a52b131890947759e9c37c74722adf8 2024-11-08T13:35:36,800 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/6c829f035b7c46ffb8dbfc9dada047f4 2024-11-08T13:35:36,805 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-08T13:35:36,805 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:35:36,805 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:36,806 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731072936701Running coprocessor pre-close hooks at 1731072936701Disabling compacts and flushes for region at 1731072936701Disabling writes for close at 1731072936701Obtaining lock to block concurrent updates at 1731072936702 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731072936702Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=3789, getHeapSize=6816, getOffHeapSize=0, getCellsCount=27 at 1731072936703 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731072936703Flushing 1588230740/info: creating writer at 1731072936703Flushing 1588230740/info: appending metadata at 1731072936737 (+34 ms)Flushing 1588230740/info: closing flushed file at 1731072936737Flushing 1588230740/ns: creating writer at 1731072936742 (+5 ms)Flushing 1588230740/ns: appending metadata at 1731072936760 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731072936760Flushing 1588230740/table: creating writer at 1731072936764 (+4 ms)Flushing 1588230740/table: appending metadata at 1731072936782 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731072936782Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3d121f74: reopening flushed file at 1731072936786 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6484c05e: reopening flushed file at 1731072936789 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55a4ba08: reopening flushed file at 1731072936793 (+4 ms)Finished flush of dataSize ~3.70 KB/3789, heapSize ~6.42 KB/6576, currentSize=0 B/0 for 1588230740 in 95ms, sequenceid=15, compaction requested=false at 1731072936797 (+4 ms)Writing region close event to WAL at 1731072936800 (+3 ms)Running coprocessor post-close hooks at 1731072936805 (+5 ms)Closed at 1731072936805 2024-11-08T13:35:36,806 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:36,901 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,35971,1731072932214; all regions closed. 2024-11-08T13:35:36,901 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,42175,1731072932076; all regions closed. 2024-11-08T13:35:36,906 DEBUG [RS:1;6ccfea20c990:35971 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:36,906 DEBUG [RS:0;6ccfea20c990:42175 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:36,906 INFO [RS:1;6ccfea20c990:35971 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C35971%2C1731072932214.meta:.meta(num 1731072933951) 2024-11-08T13:35:36,906 INFO [RS:0;6ccfea20c990:42175 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C42175%2C1731072932076:(num 1731072933593) 2024-11-08T13:35:36,906 DEBUG [RS:0;6ccfea20c990:42175 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:36,907 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:36,907 INFO [RS:0;6ccfea20c990:42175 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:35:36,907 INFO [RS:0;6ccfea20c990:42175 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T13:35:36,907 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:35:36,907 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:35:36,907 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:35:36,907 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:35:36,907 INFO [RS:0;6ccfea20c990:42175 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:35:36,908 INFO [RS:0;6ccfea20c990:42175 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42175 2024-11-08T13:35:36,908 DEBUG [RS:1;6ccfea20c990:35971 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:36,908 INFO [RS:1;6ccfea20c990:35971 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C35971%2C1731072932214:(num 1731072933595) 2024-11-08T13:35:36,908 DEBUG [RS:1;6ccfea20c990:35971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:36,908 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:36,908 INFO [RS:1;6ccfea20c990:35971 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:35:36,909 INFO [RS:1;6ccfea20c990:35971 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T13:35:36,909 INFO [RS:1;6ccfea20c990:35971 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:35:36,909 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:35:36,909 INFO [RS:1;6ccfea20c990:35971 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35971 2024-11-08T13:35:36,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:36,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,42175,1731072932076 2024-11-08T13:35:36,912 INFO [RS:0;6ccfea20c990:42175 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:35:36,912 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,35971,1731072932214 2024-11-08T13:35:36,912 INFO [RS:1;6ccfea20c990:35971 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:35:36,914 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,42175,1731072932076] 2024-11-08T13:35:36,915 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,42175,1731072932076 already deleted, retry=false 2024-11-08T13:35:36,915 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ccfea20c990,42175,1731072932076 expired; onlineServers=1 2024-11-08T13:35:36,915 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,35971,1731072932214] 2024-11-08T13:35:36,916 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,35971,1731072932214 already deleted, retry=false 2024-11-08T13:35:36,916 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ccfea20c990,35971,1731072932214 expired; onlineServers=0 2024-11-08T13:35:36,916 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6ccfea20c990,45857,1731072931375' ***** 2024-11-08T13:35:36,917 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T13:35:36,917 INFO [M:0;6ccfea20c990:45857 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:35:36,917 INFO [M:0;6ccfea20c990:45857 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:35:36,917 DEBUG [M:0;6ccfea20c990:45857 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T13:35:36,917 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T13:35:36,917 DEBUG [M:0;6ccfea20c990:45857 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T13:35:36,917 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072933214 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072933214,5,FailOnTimeoutGroup] 2024-11-08T13:35:36,917 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072933217 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072933217,5,FailOnTimeoutGroup] 2024-11-08T13:35:36,917 INFO [M:0;6ccfea20c990:45857 {}] hbase.ChoreService(370): Chore service for: master/6ccfea20c990:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T13:35:36,917 INFO [M:0;6ccfea20c990:45857 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:35:36,918 DEBUG [M:0;6ccfea20c990:45857 {}] master.HMaster(1795): Stopping service threads 2024-11-08T13:35:36,918 INFO [M:0;6ccfea20c990:45857 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T13:35:36,918 INFO [M:0;6ccfea20c990:45857 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:35:36,918 INFO [M:0;6ccfea20c990:45857 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:35:36,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T13:35:36,919 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T13:35:36,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:36,919 DEBUG [M:0;6ccfea20c990:45857 {}] zookeeper.ZKUtil(347): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T13:35:36,919 WARN [M:0;6ccfea20c990:45857 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T13:35:36,920 INFO [M:0;6ccfea20c990:45857 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.lastflushedseqids 2024-11-08T13:35:36,927 INFO [M:0;6ccfea20c990:45857 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:35:36,928 INFO [M:0;6ccfea20c990:45857 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T13:35:36,928 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:35:36,928 INFO [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:36,928 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:36,928 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:35:36,928 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:36,928 INFO [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=34.67 KB heapSize=43.02 KB 2024-11-08T13:35:36,938 DEBUG [M:0;6ccfea20c990:45857 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 30191506d0574417a81675f18e8002df 2024-11-08T13:35:36,952 DEBUG [M:0;6ccfea20c990:45857 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30191506d0574417a81675f18e8002df is 82, key is hbase:meta,,1/info:regioninfo/1731072934016/Put/seqid=0 2024-11-08T13:35:36,952 INFO [M:0;6ccfea20c990:45857 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30191506d0574417a81675f18e8002df 2024-11-08T13:35:36,960 DEBUG [M:0;6ccfea20c990:45857 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8faaf023b1bb439dbd77d18b77650590 2024-11-08T13:35:36,972 DEBUG [M:0;6ccfea20c990:45857 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8faaf023b1bb439dbd77d18b77650590 is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731072934684/Put/seqid=0 2024-11-08T13:35:36,973 INFO [M:0;6ccfea20c990:45857 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=34.05 KB at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8faaf023b1bb439dbd77d18b77650590 2024-11-08T13:35:36,982 DEBUG [M:0;6ccfea20c990:45857 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on ffe2df65d94b4448822426ecbd7f8025 2024-11-08T13:35:36,993 DEBUG [M:0;6ccfea20c990:45857 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ffe2df65d94b4448822426ecbd7f8025 is 69, key is 6ccfea20c990,35971,1731072932214/rs:state/1731072933337/Put/seqid=0 2024-11-08T13:35:36,993 INFO [M:0;6ccfea20c990:45857 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=84 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ffe2df65d94b4448822426ecbd7f8025 2024-11-08T13:35:36,995 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/30191506d0574417a81675f18e8002df as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/30191506d0574417a81675f18e8002df 2024-11-08T13:35:36,996 INFO [M:0;6ccfea20c990:45857 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/30191506d0574417a81675f18e8002df, entries=8, sequenceid=84, filesize=5.5 K 2024-11-08T13:35:36,997 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/8faaf023b1bb439dbd77d18b77650590 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8faaf023b1bb439dbd77d18b77650590 2024-11-08T13:35:36,998 INFO [M:0;6ccfea20c990:45857 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/8faaf023b1bb439dbd77d18b77650590, entries=9, sequenceid=84, filesize=6.5 K 2024-11-08T13:35:36,999 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ffe2df65d94b4448822426ecbd7f8025 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ffe2df65d94b4448822426ecbd7f8025 2024-11-08T13:35:37,001 INFO [M:0;6ccfea20c990:45857 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ffe2df65d94b4448822426ecbd7f8025, entries=2, sequenceid=84, filesize=5.1 K 2024-11-08T13:35:37,002 INFO [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(3140): Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=84, compaction requested=false 2024-11-08T13:35:37,004 INFO [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:37,004 DEBUG [M:0;6ccfea20c990:45857 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731072936928Disabling compacts and flushes for region at 1731072936928Disabling writes for close at 1731072936928Obtaining lock to block concurrent updates at 1731072936928Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731072936928Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=35506, getHeapSize=43992, getOffHeapSize=0, getCellsCount=99 at 1731072936929 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731072936929Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731072936929Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731072936951 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731072936951Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731072936954 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731072936972 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731072936972Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731072936975 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731072936992 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731072936992Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@e1d7f5e: reopening flushed file at 1731072936995 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e12b836: reopening flushed file at 1731072936997 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48e80c52: reopening flushed file at 1731072936998 (+1 ms)Finished flush of dataSize ~34.67 KB/35506, heapSize ~42.73 KB/43752, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 74ms, sequenceid=84, compaction requested=false at 1731072937002 (+4 ms)Writing region close event to WAL at 1731072937004 (+2 ms)Closed at 1731072937004 2024-11-08T13:35:37,006 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:35:37,006 INFO [M:0;6ccfea20c990:45857 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T13:35:37,006 INFO [M:0;6ccfea20c990:45857 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45857 2024-11-08T13:35:37,006 INFO [M:0;6ccfea20c990:45857 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:35:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35971-0x1001f8c116e0002, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:37,014 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42175-0x1001f8c116e0001, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:37,015 INFO [RS:0;6ccfea20c990:42175 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:35:37,015 INFO [RS:1;6ccfea20c990:35971 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:35:37,015 INFO [RS:0;6ccfea20c990:42175 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,42175,1731072932076; zookeeper connection closed. 2024-11-08T13:35:37,015 INFO [RS:1;6ccfea20c990:35971 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,35971,1731072932214; zookeeper connection closed. 2024-11-08T13:35:37,015 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6e0841ef {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6e0841ef 2024-11-08T13:35:37,015 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@3397505c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@3397505c 2024-11-08T13:35:37,016 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-08T13:35:37,108 INFO [M:0;6ccfea20c990:45857 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:35:37,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:37,108 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45857-0x1001f8c116e0000, quorum=127.0.0.1:65101, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:37,112 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T13:35:37,132 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T13:35:37,133 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,141 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnRegionMove Thread=49 (was 12) Potentially hanging thread: master/6ccfea20c990:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: regionserver/6ccfea20c990:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731072936624 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731072935052 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731072936791 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master/6ccfea20c990:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: hfile-prefetch-1731072936788 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1177) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/6ccfea20c990:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-4-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=346 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=234 (was 185) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3403 (was 3740) 2024-11-08T13:35:37,146 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=49, OpenFileDescriptor=346, MaxFileDescriptor=1048576, SystemLoadAverage=234, ProcessCount=11, AvailableMemoryMB=3403 2024-11-08T13:35:37,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/hadoop-log-dir so I do NOT create it in target/test-data/27554775-3432-897e-40ce-f29aa148dcd2 2024-11-08T13:35:37,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/hadoop-tmp-dir so I do NOT create it in target/test-data/27554775-3432-897e-40ce-f29aa148dcd2 2024-11-08T13:35:37,146 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,149 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/cluster_b745403b-6e59-5217-1162-19d9452384d6, deleteOnExit=true 2024-11-08T13:35:37,152 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/cluster_b745403b-6e59-5217-1162-19d9452384d6/zookeeper_0, clientPort=51310, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/cluster_b745403b-6e59-5217-1162-19d9452384d6/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/cluster_b745403b-6e59-5217-1162-19d9452384d6/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T13:35:37,153 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=51310 2024-11-08T13:35:37,163 INFO [Time-limited test {}] util.FSUtils(489): Created version file at /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a with version=8 2024-11-08T13:35:37,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/hbase-staging 2024-11-08T13:35:37,166 INFO [Time-limited test {}] client.ConnectionUtils(128): master/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:35:37,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,166 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:35:37,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,166 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:35:37,166 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T13:35:37,167 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:35:37,167 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41401 2024-11-08T13:35:37,169 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41401 connecting to ZooKeeper ensemble=127.0.0.1:51310 2024-11-08T13:35:37,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:414010x0, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:35:37,174 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41401-0x1001f8c292e0000 connected 2024-11-08T13:35:37,186 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,186 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,187 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:37,187 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a, hbase.cluster.distributed=false 2024-11-08T13:35:37,189 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:35:37,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41401 2024-11-08T13:35:37,190 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41401 2024-11-08T13:35:37,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41401 2024-11-08T13:35:37,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41401 2024-11-08T13:35:37,191 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41401 2024-11-08T13:35:37,209 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:35:37,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,209 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:35:37,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,209 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:35:37,209 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:35:37,209 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:35:37,210 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36185 2024-11-08T13:35:37,212 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36185 connecting to ZooKeeper ensemble=127.0.0.1:51310 2024-11-08T13:35:37,212 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,212 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361850x0, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:35:37,215 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361850x0, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:37,216 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:35:37,219 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36185-0x1001f8c292e0001 connected 2024-11-08T13:35:37,220 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.cache 2024-11-08T13:35:37,220 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T13:35:37,220 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T13:35:37,220 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T13:35:37,221 INFO [Thread-118 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T13:35:37,221 WARN [Thread-118 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T13:35:37,221 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T13:35:37,222 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:35:37,222 INFO [Thread-118 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T13:35:37,222 INFO [Thread-118 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence is complete. 2024-11-08T13:35:37,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:35:37,224 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:35:37,225 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36185 2024-11-08T13:35:37,227 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36185 2024-11-08T13:35:37,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36185 2024-11-08T13:35:37,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36185 2024-11-08T13:35:37,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36185 2024-11-08T13:35:37,249 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:35:37,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,249 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:35:37,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:37,249 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:35:37,249 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:35:37,249 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:35:37,250 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33005 2024-11-08T13:35:37,251 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33005 connecting to ZooKeeper ensemble=127.0.0.1:51310 2024-11-08T13:35:37,251 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,252 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,254 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:330050x0, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:35:37,254 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:37,254 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33005-0x1001f8c292e0002 connected 2024-11-08T13:35:37,255 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:35:37,255 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.cache 2024-11-08T13:35:37,256 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T13:35:37,256 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T13:35:37,256 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T13:35:37,256 INFO [Thread-119 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T13:35:37,256 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T13:35:37,256 WARN [Thread-119 {}] bucket.BucketCache(1534): Persistence file missing! It's ok if it's first run after enabling persistent cache. 2024-11-08T13:35:37,256 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:35:37,256 INFO [Thread-119 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T13:35:37,257 INFO [Thread-119 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence is complete. 2024-11-08T13:35:37,257 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T13:35:37,259 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:35:37,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33005 2024-11-08T13:35:37,264 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33005 2024-11-08T13:35:37,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33005 2024-11-08T13:35:37,265 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33005 2024-11-08T13:35:37,269 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33005 2024-11-08T13:35:37,281 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;6ccfea20c990:41401 2024-11-08T13:35:37,281 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/6ccfea20c990,41401,1731072937165 2024-11-08T13:35:37,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:37,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:37,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:37,285 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/6ccfea20c990,41401,1731072937165 2024-11-08T13:35:37,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:35:37,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T13:35:37,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,286 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,287 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T13:35:37,287 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/6ccfea20c990,41401,1731072937165 from backup master directory 2024-11-08T13:35:37,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/6ccfea20c990,41401,1731072937165 2024-11-08T13:35:37,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:37,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:37,288 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T13:35:37,289 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:35:37,289 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=6ccfea20c990,41401,1731072937165 2024-11-08T13:35:37,290 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/hbase.id] with ID: b1b5bdb3-88eb-4e63-8d69-b0bf7d219f73 2024-11-08T13:35:37,290 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.tmp/hbase.id 2024-11-08T13:35:37,298 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/hbase.id] 2024-11-08T13:35:37,334 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:37,334 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T13:35:37,336 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-08T13:35:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,339 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,344 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:35:37,346 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T13:35:37,347 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:37,365 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store 2024-11-08T13:35:37,371 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:37,371 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:35:37,371 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:37,372 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:37,372 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:35:37,372 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:37,372 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:37,372 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731072937371Disabling compacts and flushes for region at 1731072937371Disabling writes for close at 1731072937372 (+1 ms)Writing region close event to WAL at 1731072937372Closed at 1731072937372 2024-11-08T13:35:37,372 WARN [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/.initializing 2024-11-08T13:35:37,372 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/WALs/6ccfea20c990,41401,1731072937165 2024-11-08T13:35:37,378 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:35:37,380 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=6ccfea20c990%2C41401%2C1731072937165, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/WALs/6ccfea20c990,41401,1731072937165, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/oldWALs, maxLogs=10 2024-11-08T13:35:37,396 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/WALs/6ccfea20c990,41401,1731072937165/6ccfea20c990%2C41401%2C1731072937165.1731072937381 2024-11-08T13:35:37,397 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:37,397 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:37,397 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:37,397 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,397 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,398 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T13:35:37,402 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:37,402 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,405 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T13:35:37,406 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:37,406 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,409 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T13:35:37,409 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,409 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:37,409 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T13:35:37,412 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,412 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:37,413 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,413 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,413 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,413 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,413 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,414 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:35:37,414 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T13:35:37,419 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:35:37,419 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72459935, jitterRate=0.07973717153072357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:35:37,420 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731072937398Initializing all the Stores at 1731072937398Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072937398Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072937398Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072937398Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072937398Cleaning up temporary data from old regions at 1731072937413 (+15 ms)Region opened successfully at 1731072937420 (+7 ms) 2024-11-08T13:35:37,421 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T13:35:37,424 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@efcf08f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:35:37,425 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T13:35:37,425 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T13:35:37,426 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T13:35:37,426 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T13:35:37,426 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-08T13:35:37,426 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-08T13:35:37,427 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T13:35:37,428 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T13:35:37,429 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T13:35:37,430 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T13:35:37,431 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T13:35:37,431 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T13:35:37,432 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T13:35:37,432 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T13:35:37,433 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T13:35:37,434 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T13:35:37,435 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T13:35:37,435 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T13:35:37,437 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T13:35:37,438 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T13:35:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:37,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,440 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=6ccfea20c990,41401,1731072937165, sessionid=0x1001f8c292e0000, setting cluster-up flag (Was=false) 2024-11-08T13:35:37,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,442 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,445 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T13:35:37,446 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,41401,1731072937165 2024-11-08T13:35:37,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,452 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T13:35:37,453 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=6ccfea20c990,41401,1731072937165 2024-11-08T13:35:37,453 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T13:35:37,455 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:37,455 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T13:35:37,455 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T13:35:37,456 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 6ccfea20c990,41401,1731072937165 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T13:35:37,457 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:37,457 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:37,457 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:37,457 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/6ccfea20c990:0, corePoolSize=5, maxPoolSize=5 2024-11-08T13:35:37,457 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/6ccfea20c990:0, corePoolSize=10, maxPoolSize=10 2024-11-08T13:35:37,457 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,458 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:35:37,458 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,458 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731072967458 2024-11-08T13:35:37,459 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T13:35:37,459 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T13:35:37,459 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T13:35:37,459 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T13:35:37,459 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T13:35:37,459 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T13:35:37,459 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:37,459 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,459 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T13:35:37,460 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,460 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T13:35:37,461 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T13:35:37,461 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T13:35:37,461 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T13:35:37,461 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T13:35:37,461 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T13:35:37,462 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072937462,5,FailOnTimeoutGroup] 2024-11-08T13:35:37,462 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072937462,5,FailOnTimeoutGroup] 2024-11-08T13:35:37,462 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,462 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T13:35:37,462 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,462 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,470 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T13:35:37,471 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:37,471 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(746): ClusterId : b1b5bdb3-88eb-4e63-8d69-b0bf7d219f73 2024-11-08T13:35:37,471 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:35:37,472 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(746): ClusterId : b1b5bdb3-88eb-4e63-8d69-b0bf7d219f73 2024-11-08T13:35:37,472 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:35:37,473 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:35:37,473 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:35:37,474 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:35:37,475 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:35:37,476 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:35:37,476 DEBUG [RS:1;6ccfea20c990:33005 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cdfcdb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:35:37,477 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:37,477 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T13:35:37,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:35:37,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:37,481 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T13:35:37,481 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:35:37,482 DEBUG [RS:0;6ccfea20c990:36185 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358a4961, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:35:37,484 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:35:37,484 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:37,485 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T13:35:37,490 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:35:37,490 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:37,490 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T13:35:37,492 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;6ccfea20c990:33005 2024-11-08T13:35:37,492 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:35:37,492 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:35:37,492 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:35:37,494 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,41401,1731072937165 with port=33005, startcode=1731072937248 2024-11-08T13:35:37,494 DEBUG [RS:1;6ccfea20c990:33005 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:35:37,495 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:35:37,495 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:37,495 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:37,496 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:35:37,496 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:37,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:37,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:35:37,497 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:35:37,497 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:35:37,497 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37255, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:35:37,498 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:35:37,498 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;6ccfea20c990:36185 2024-11-08T13:35:37,498 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:35:37,498 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:35:37,498 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:35:37,498 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41401 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,33005,1731072937248 2024-11-08T13:35:37,499 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41401 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,33005,1731072937248 2024-11-08T13:35:37,499 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,41401,1731072937165 with port=36185, startcode=1731072937208 2024-11-08T13:35:37,500 DEBUG [RS:0;6ccfea20c990:36185 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:35:37,500 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:37,500 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T13:35:37,501 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:35:37,502 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:37,503 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57833, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:35:37,503 DEBUG [RS:1;6ccfea20c990:33005 {}] zookeeper.ZKUtil(111): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,33005,1731072937248 2024-11-08T13:35:37,503 WARN [RS:1;6ccfea20c990:33005 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:35:37,503 INFO [RS:1;6ccfea20c990:33005 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:37,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41401 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,36185,1731072937208 2024-11-08T13:35:37,503 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248 2024-11-08T13:35:37,503 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41401 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,36185,1731072937208 2024-11-08T13:35:37,505 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:37,505 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T13:35:37,505 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:35:37,506 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,33005,1731072937248] 2024-11-08T13:35:37,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:37,508 DEBUG [RS:0;6ccfea20c990:36185 {}] zookeeper.ZKUtil(111): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,36185,1731072937208 2024-11-08T13:35:37,508 WARN [RS:0;6ccfea20c990:36185 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:35:37,508 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,36185,1731072937208] 2024-11-08T13:35:37,508 INFO [RS:0;6ccfea20c990:36185 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:37,509 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:35:37,509 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,36185,1731072937208 2024-11-08T13:35:37,509 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64610162, jitterRate=-0.037233561277389526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:35:37,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731072937477Initializing all the Stores at 1731072937477Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072937477Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072937477Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072937477Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072937477Cleaning up temporary data from old regions at 1731072937497 (+20 ms)Region opened successfully at 1731072937510 (+13 ms) 2024-11-08T13:35:37,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:35:37,510 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:35:37,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:35:37,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:35:37,510 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:35:37,513 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:37,514 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731072937510Disabling compacts and flushes for region at 1731072937510Disabling writes for close at 1731072937510Writing region close event to WAL at 1731072937513 (+3 ms)Closed at 1731072937513 2024-11-08T13:35:37,514 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:37,514 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T13:35:37,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T13:35:37,516 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:35:37,517 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T13:35:37,518 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:35:37,521 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:35:37,521 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:35:37,521 INFO [RS:0;6ccfea20c990:36185 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:35:37,521 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,523 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:35:37,523 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:35:37,523 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:35:37,523 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,523 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,523 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,523 INFO [RS:1;6ccfea20c990:33005 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:35:37,524 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,524 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,525 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:37,525 DEBUG [RS:0;6ccfea20c990:36185 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,525 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,526 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,526 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,526 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,526 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:37,526 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:37,526 DEBUG [RS:1;6ccfea20c990:33005 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:37,526 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,526 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,36185,1731072937208-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,527 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,528 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,33005,1731072937248-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:35:37,541 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:35:37,542 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,36185,1731072937208-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,542 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,542 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.Replication(171): 6ccfea20c990,36185,1731072937208 started 2024-11-08T13:35:37,543 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:35:37,543 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,33005,1731072937248-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,543 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,543 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.Replication(171): 6ccfea20c990,33005,1731072937248 started 2024-11-08T13:35:37,555 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,556 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,36185,1731072937208, RpcServer on 6ccfea20c990/172.17.0.2:36185, sessionid=0x1001f8c292e0001 2024-11-08T13:35:37,556 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:35:37,556 DEBUG [RS:0;6ccfea20c990:36185 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,36185,1731072937208 2024-11-08T13:35:37,556 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,36185,1731072937208' 2024-11-08T13:35:37,556 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:35:37,556 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:35:37,557 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:35:37,557 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:35:37,557 DEBUG [RS:0;6ccfea20c990:36185 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,36185,1731072937208 2024-11-08T13:35:37,557 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,36185,1731072937208' 2024-11-08T13:35:37,557 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:35:37,557 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:35:37,558 DEBUG [RS:0;6ccfea20c990:36185 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:35:37,558 INFO [RS:0;6ccfea20c990:36185 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:35:37,558 INFO [RS:0;6ccfea20c990:36185 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:35:37,558 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:37,558 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,33005,1731072937248, RpcServer on 6ccfea20c990/172.17.0.2:33005, sessionid=0x1001f8c292e0002 2024-11-08T13:35:37,559 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:35:37,559 DEBUG [RS:1;6ccfea20c990:33005 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,33005,1731072937248 2024-11-08T13:35:37,559 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,33005,1731072937248' 2024-11-08T13:35:37,559 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:35:37,559 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:35:37,560 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:35:37,560 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:35:37,560 DEBUG [RS:1;6ccfea20c990:33005 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,33005,1731072937248 2024-11-08T13:35:37,560 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,33005,1731072937248' 2024-11-08T13:35:37,560 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:35:37,560 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:35:37,561 DEBUG [RS:1;6ccfea20c990:33005 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:35:37,561 INFO [RS:1;6ccfea20c990:33005 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:35:37,561 INFO [RS:1;6ccfea20c990:33005 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:35:37,659 INFO [RS:0;6ccfea20c990:36185 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:35:37,661 INFO [RS:0;6ccfea20c990:36185 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C36185%2C1731072937208, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,36185,1731072937208, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:37,662 INFO [RS:1;6ccfea20c990:33005 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:35:37,664 INFO [RS:1;6ccfea20c990:33005 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C33005%2C1731072937248, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:37,667 WARN [6ccfea20c990:41401 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T13:35:37,684 INFO [RS:1;6ccfea20c990:33005 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248/6ccfea20c990%2C33005%2C1731072937248.1731072937668 2024-11-08T13:35:37,685 DEBUG [RS:1;6ccfea20c990:33005 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:37,688 INFO [RS:0;6ccfea20c990:36185 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,36185,1731072937208/6ccfea20c990%2C36185%2C1731072937208.1731072937669 2024-11-08T13:35:37,689 DEBUG [RS:0;6ccfea20c990:36185 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:37,917 DEBUG [6ccfea20c990:41401 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-08T13:35:37,918 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:35:37,918 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:35:37,918 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:35:37,918 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:35:37,918 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:35:37,918 INFO [6ccfea20c990:41401 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:35:37,918 INFO [6ccfea20c990:41401 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:35:37,918 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:35:37,919 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ccfea20c990,33005,1731072937248 2024-11-08T13:35:37,920 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,33005,1731072937248, state=OPENING 2024-11-08T13:35:37,921 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T13:35:37,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:37,923 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:35:37,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:37,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:37,923 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:37,923 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,33005,1731072937248}] 2024-11-08T13:35:38,076 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:35:38,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38437, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:35:38,085 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T13:35:38,086 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:38,087 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T13:35:38,088 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C33005%2C1731072937248.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:38,109 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248/6ccfea20c990%2C33005%2C1731072937248.meta.1731072938090.meta 2024-11-08T13:35:38,110 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:38,110 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:38,110 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T13:35:38,110 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T13:35:38,110 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T13:35:38,111 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T13:35:38,111 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:38,111 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T13:35:38,111 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T13:35:38,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T13:35:38,112 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:35:38,112 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:38,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:38,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T13:35:38,113 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:35:38,113 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:38,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:38,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T13:35:38,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:35:38,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:38,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:38,114 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T13:35:38,115 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:35:38,115 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:38,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:38,115 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:35:38,116 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:38,116 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:38,116 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:35:38,116 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:35:38,116 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:35:38,117 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:35:38,117 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71679510, jitterRate=0.06810793280601501}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:35:38,118 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T13:35:38,118 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731072938111Writing region info on filesystem at 1731072938111Initializing all the Stores at 1731072938111Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072938111Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072938111Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072938111Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072938111Cleaning up temporary data from old regions at 1731072938116 (+5 ms)Running coprocessor post-open hooks at 1731072938118 (+2 ms)Region opened successfully at 1731072938118 2024-11-08T13:35:38,119 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731072938076 2024-11-08T13:35:38,122 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T13:35:38,122 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T13:35:38,122 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,33005,1731072937248 2024-11-08T13:35:38,123 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,33005,1731072937248, state=OPEN 2024-11-08T13:35:38,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:38,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:38,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:38,124 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ccfea20c990,33005,1731072937248 2024-11-08T13:35:38,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:38,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:38,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:38,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T13:35:38,127 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,33005,1731072937248 in 201 msec 2024-11-08T13:35:38,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T13:35:38,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 612 msec 2024-11-08T13:35:38,129 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T13:35:38,129 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T13:35:38,132 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:35:38,132 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,33005,1731072937248, seqNum=-1] 2024-11-08T13:35:38,133 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:35:38,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47167, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:35:38,140 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 685 msec 2024-11-08T13:35:38,140 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731072938140, completionTime=-1 2024-11-08T13:35:38,140 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=2; waited=0ms, expected min=2 server(s), max=2 server(s), master is running 2024-11-08T13:35:38,140 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T13:35:38,143 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-08T13:35:38,143 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731072998143 2024-11-08T13:35:38,143 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731073058143 2024-11-08T13:35:38,143 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-08T13:35:38,143 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41401,1731072937165-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:38,143 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41401,1731072937165-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:38,144 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41401,1731072937165-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:38,144 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-6ccfea20c990:41401, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:38,144 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:38,144 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:38,146 DEBUG [master/6ccfea20c990:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T13:35:38,149 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.860sec 2024-11-08T13:35:38,149 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T13:35:38,149 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T13:35:38,149 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T13:35:38,149 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T13:35:38,150 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T13:35:38,150 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41401,1731072937165-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:35:38,150 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41401,1731072937165-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T13:35:38,150 DEBUG [master/6ccfea20c990:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T13:35:38,151 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T13:35:38,151 INFO [master/6ccfea20c990:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41401,1731072937165-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:38,172 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@689405c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:35:38,172 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 6ccfea20c990,41401,-1 for getting cluster id 2024-11-08T13:35:38,173 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T13:35:38,175 DEBUG [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b1b5bdb3-88eb-4e63-8d69-b0bf7d219f73' 2024-11-08T13:35:38,175 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T13:35:38,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b1b5bdb3-88eb-4e63-8d69-b0bf7d219f73" 2024-11-08T13:35:38,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5976a4e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:35:38,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [6ccfea20c990,41401,-1] 2024-11-08T13:35:38,176 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T13:35:38,177 DEBUG [RPCClient-NioEventLoopGroup-5-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:38,178 INFO [HMaster-EventLoopGroup-6-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T13:35:38,180 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d9d15ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T13:35:38,180 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:35:38,182 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,33005,1731072937248, seqNum=-1] 2024-11-08T13:35:38,182 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:35:38,185 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:35:38,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=6ccfea20c990,41401,1731072937165 2024-11-08T13:35:38,188 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T13:35:38,189 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 6ccfea20c990,41401,1731072937165 2024-11-08T13:35:38,190 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@20442acc 2024-11-08T13:35:38,190 DEBUG [RPCClient-NioEventLoopGroup-5-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T13:35:38,193 INFO [HMaster-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T13:35:38,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testBlockEvictionOnGracefulStop1731072938187', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T13:35:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 2024-11-08T13:35:38,196 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T13:35:38,196 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:38,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testBlockEvictionOnGracefulStop1731072938187" procId is: 4 2024-11-08T13:35:38,197 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T13:35:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:38,211 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dcb92de9a01da820456a423aff5a6cc4, NAME => 'testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testBlockEvictionOnGracefulStop1731072938187', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:38,217 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:38,217 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(1722): Closing dcb92de9a01da820456a423aff5a6cc4, disabling compactions & flushes 2024-11-08T13:35:38,217 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:38,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:38,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. after waiting 0 ms 2024-11-08T13:35:38,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:38,218 INFO [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:38,218 DEBUG [RegionOpenAndInit-testBlockEvictionOnGracefulStop1731072938187-pool-0 {}] regionserver.HRegion(1676): Region close journal for dcb92de9a01da820456a423aff5a6cc4: Waiting for close lock at 1731072938217Disabling compacts and flushes for region at 1731072938217Disabling writes for close at 1731072938218 (+1 ms)Writing region close event to WAL at 1731072938218Closed at 1731072938218 2024-11-08T13:35:38,219 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T13:35:38,219 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.","families":{"info":[{"qualifier":"regioninfo","vlen":78,"tag":[],"timestamp":"1731072938219"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731072938219"}]},"ts":"1731072938219"} 2024-11-08T13:35:38,222 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T13:35:38,222 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T13:35:38,223 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731072938187","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731072938223"}]},"ts":"1731072938223"} 2024-11-08T13:35:38,225 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731072938187, state=ENABLING in hbase:meta 2024-11-08T13:35:38,225 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:35:38,225 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:35:38,226 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:35:38,226 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:35:38,226 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:35:38,226 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:35:38,226 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:35:38,226 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:35:38,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN}] 2024-11-08T13:35:38,227 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN 2024-11-08T13:35:38,228 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN; state=OFFLINE, location=6ccfea20c990,33005,1731072937248; forceNewPlan=false, retain=false 2024-11-08T13:35:38,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:38,379 INFO [6ccfea20c990:41401 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-08T13:35:38,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dcb92de9a01da820456a423aff5a6cc4, regionState=OPENING, regionLocation=6ccfea20c990,33005,1731072937248 2024-11-08T13:35:38,384 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN because future has completed 2024-11-08T13:35:38,386 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcb92de9a01da820456a423aff5a6cc4, server=6ccfea20c990,33005,1731072937248}] 2024-11-08T13:35:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:38,541 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:38,542 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => dcb92de9a01da820456a423aff5a6cc4, NAME => 'testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:38,542 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731072938187 dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,542 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:38,542 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,542 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,543 INFO [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,546 INFO [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcb92de9a01da820456a423aff5a6cc4 columnFamilyName family 2024-11-08T13:35:38,546 DEBUG [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:38,546 INFO [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] regionserver.HStore(327): Store=dcb92de9a01da820456a423aff5a6cc4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:38,547 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,547 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,547 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,547 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,547 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,548 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,553 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T13:35:38,553 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened dcb92de9a01da820456a423aff5a6cc4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73128391, jitterRate=0.0896979421377182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:35:38,553 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:38,554 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for dcb92de9a01da820456a423aff5a6cc4: Running coprocessor pre-open hook at 1731072938543Writing region info on filesystem at 1731072938543Initializing all the Stores at 1731072938543Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072938543Cleaning up temporary data from old regions at 1731072938548 (+5 ms)Running coprocessor post-open hooks at 1731072938553 (+5 ms)Region opened successfully at 1731072938554 (+1 ms) 2024-11-08T13:35:38,555 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4., pid=6, masterSystemTime=1731072938539 2024-11-08T13:35:38,557 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:38,557 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:38,558 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=dcb92de9a01da820456a423aff5a6cc4, regionState=OPEN, openSeqNum=2, regionLocation=6ccfea20c990,33005,1731072937248 2024-11-08T13:35:38,560 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcb92de9a01da820456a423aff5a6cc4, server=6ccfea20c990,33005,1731072937248 because future has completed 2024-11-08T13:35:38,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T13:35:38,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure dcb92de9a01da820456a423aff5a6cc4, server=6ccfea20c990,33005,1731072937248 in 176 msec 2024-11-08T13:35:38,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T13:35:38,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN in 337 msec 2024-11-08T13:35:38,565 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T13:35:38,566 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testBlockEvictionOnGracefulStop1731072938187","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731072938565"}]},"ts":"1731072938565"} 2024-11-08T13:35:38,568 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testBlockEvictionOnGracefulStop1731072938187, state=ENABLED in hbase:meta 2024-11-08T13:35:38,569 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T13:35:38,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testBlockEvictionOnGracefulStop1731072938187 in 375 msec 2024-11-08T13:35:38,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41401 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T13:35:38,832 INFO [RPCClient-NioEventLoopGroup-5-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testBlockEvictionOnGracefulStop1731072938187 completed 2024-11-08T13:35:38,832 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testBlockEvictionOnGracefulStop1731072938187 get assigned. Timeout = 60000ms 2024-11-08T13:35:38,832 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:35:38,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testBlockEvictionOnGracefulStop1731072938187 assigned to meta. Checking AM states. 2024-11-08T13:35:38,843 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-08T13:35:38,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testBlockEvictionOnGracefulStop1731072938187 assigned. 2024-11-08T13:35:38,848 DEBUG [RPCClient-NioEventLoopGroup-5-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testBlockEvictionOnGracefulStop1731072938187', row='row1', locateType=CURRENT is [region=testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4., hostname=6ccfea20c990,33005,1731072937248, seqNum=2] 2024-11-08T13:35:38,852 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dcb92de9a01da820456a423aff5a6cc4 1/1 column families, dataSize=78 B heapSize=480 B 2024-11-08T13:35:38,860 DEBUG [Time-limited test {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6c5753710d524e958fe66e44c82dfd87 2024-11-08T13:35:38,872 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/.tmp/family/6c5753710d524e958fe66e44c82dfd87 is 43, key is row1/family:qf1/1/Put/seqid=0 2024-11-08T13:35:38,873 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/.tmp/family/6c5753710d524e958fe66e44c82dfd87 2024-11-08T13:35:38,876 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/.tmp/family/6c5753710d524e958fe66e44c82dfd87 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87 2024-11-08T13:35:38,877 DEBUG [Time-limited test {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87, delay=983 ms 2024-11-08T13:35:38,878 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87, entries=2, sequenceid=6, filesize=4.9 K 2024-11-08T13:35:38,879 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~78 B/78, heapSize ~464 B/464, currentSize=0 B/0 for dcb92de9a01da820456a423aff5a6cc4 in 27ms, sequenceid=6, compaction requested=false 2024-11-08T13:35:38,879 DEBUG [Time-limited test {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testBlockEvictionOnGracefulStop1731072938187' 2024-11-08T13:35:38,880 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dcb92de9a01da820456a423aff5a6cc4: 2024-11-08T13:35:39,486 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T13:35:39,861 DEBUG [hfile-prefetch-1731072936624 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6c5753710d524e958fe66e44c82dfd87_0. 2024-11-08T13:35:39,861 DEBUG [hfile-prefetch-1731072936624 {}] hfile.HFilePreadReader$1(107): No entry in the backing map for cache key 6c5753710d524e958fe66e44c82dfd87_117. 2024-11-08T13:35:39,862 DEBUG [hfile-prefetch-1731072936624 {}] bucket.BucketCache(2282): Notifying caching completed for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87, with total blocks 2, and data blocks 1 2024-11-08T13:35:39,862 DEBUG [hfile-prefetch-1731072936624 {}] bucket.BucketCache(2286): iterating over 0 entries in the backing map 2024-11-08T13:35:39,862 DEBUG [hfile-prefetch-1731072936624 {}] bucket.BucketCache(2309): Prefetch executor completed for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87, but only 0 data blocks were cached. Total data blocks for file: 1. Checking for blocks pending cache in cache writer queue. 2024-11-08T13:35:39,862 DEBUG [hfile-prefetch-1731072936624 {}] bucket.BucketCache(2319): There are still blocks pending caching for file file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87. Will sleep 100ms and try the verification again. 2024-11-08T13:35:39,880 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(504): Stopping Thread[RS:1;6ccfea20c990:33005,5,FailOnTimeoutGroup] 2024-11-08T13:35:39,880 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,33005,1731072937248' ***** 2024-11-08T13:35:39,880 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Stopping rs 1 2024-11-08T13:35:39,881 INFO [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T13:35:39,881 INFO [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence 2024-11-08T13:35:39,881 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T13:35:39,881 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T13:35:39,881 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T13:35:39,882 INFO [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T13:35:39,882 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T13:35:39,892 DEBUG [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1488): Thread RS:1;6ccfea20c990:33005 started persisting bucket cache to file 2024-11-08T13:35:39,892 DEBUG [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:39,892 DEBUG [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 1, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:39,895 DEBUG [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 1, fullycachedFiles size: 0 2024-11-08T13:35:39,895 DEBUG [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 1, fullycachedFiles size: 0, file name: bucket.persistence1731072939892 2024-11-08T13:35:39,895 DEBUG [RS:1;6ccfea20c990:33005 {}] bucket.BucketCache(1510): Thread RS:1;6ccfea20c990:33005 finished persisting bucket cache to file, renaming 2024-11-08T13:35:39,895 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:35:39,895 INFO [RS:1;6ccfea20c990:33005 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:35:39,895 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:35:39,895 INFO [RS:1;6ccfea20c990:33005 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:35:39,895 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(3091): Received CLOSE for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:39,895 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,33005,1731072937248 2024-11-08T13:35:39,895 INFO [RS:1;6ccfea20c990:33005 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:35:39,895 INFO [RS:1;6ccfea20c990:33005 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;6ccfea20c990:33005. 2024-11-08T13:35:39,895 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T13:35:39,895 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dcb92de9a01da820456a423aff5a6cc4, disabling compactions & flushes 2024-11-08T13:35:39,896 DEBUG [RS:1;6ccfea20c990:33005 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:39,896 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:39,896 DEBUG [RS:1;6ccfea20c990:33005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:39,896 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:39,896 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. after waiting 0 ms 2024-11-08T13:35:39,896 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:35:39,896 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:39,896 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:35:39,896 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:35:39,896 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T13:35:39,896 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T13:35:39,896 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, dcb92de9a01da820456a423aff5a6cc4=testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.} 2024-11-08T13:35:39,896 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T13:35:39,896 DEBUG [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:39,896 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:35:39,896 DEBUG [StoreFileCloser-dcb92de9a01da820456a423aff5a6cc4-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87 2024-11-08T13:35:39,896 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:35:39,896 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:35:39,896 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:35:39,897 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:35:39,896 WARN [hfile-prefetch-1731072936624 {}] hfile.HFilePreadReader$1(158): Prefetch path=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87, offset=158, end=158 java.lang.RuntimeException: java.lang.InterruptedException: sleep interrupted at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2338) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.CombinedBlockCache.notifyFileCachingCompleted(CombinedBlockCache.java:469) ~[classes/:?] at org.apache.hadoop.hbase.io.hfile.HFilePreadReader$1.run(HFilePreadReader.java:148) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.notifyFileCachingCompleted(BucketCache.java:2321) ~[classes/:?] ... 9 more 2024-11-08T13:35:39,897 DEBUG [hfile-prefetch-1731072936624 {}] hfile.PrefetchExecutor(121): Prefetch completed for 6c5753710d524e958fe66e44c82dfd87 2024-11-08T13:35:39,897 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.75 KB heapSize=3.78 KB 2024-11-08T13:35:39,902 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-08T13:35:39,902 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:39,903 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dcb92de9a01da820456a423aff5a6cc4: Waiting for close lock at 1731072939895Running coprocessor pre-close hooks at 1731072939895Disabling compacts and flushes for region at 1731072939895Disabling writes for close at 1731072939896 (+1 ms)Writing region close event to WAL at 1731072939897 (+1 ms)Running coprocessor post-close hooks at 1731072939902 (+5 ms)Closed at 1731072939902 2024-11-08T13:35:39,903 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:39,908 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 6cc437e82d074fe2aa6fa7083afd5a7b 2024-11-08T13:35:39,923 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/6cc437e82d074fe2aa6fa7083afd5a7b is 209, key is testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4./info:regioninfo/1731072938557/Put/seqid=0 2024-11-08T13:35:39,924 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.53 KB at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/6cc437e82d074fe2aa6fa7083afd5a7b 2024-11-08T13:35:39,925 DEBUG [bucket-cache-persister {}] bucket.BucketCachePersister(47): Cache is inconsistent, persisting to disk 2024-11-08T13:35:39,925 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1488): Thread bucket-cache-persister started persisting bucket cache to file 2024-11-08T13:35:39,925 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:39,925 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:39,928 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T13:35:39,928 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731072939925 2024-11-08T13:35:39,928 DEBUG [bucket-cache-persister {}] bucket.BucketCache(1510): Thread bucket-cache-persister finished persisting bucket cache to file, renaming 2024-11-08T13:35:39,930 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:39,931 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 96ab2eced20940f3a77a56aab32725eb 2024-11-08T13:35:39,943 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/ns/96ab2eced20940f3a77a56aab32725eb is 43, key is default/ns:d/1731072938135/Put/seqid=0 2024-11-08T13:35:39,943 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/ns/96ab2eced20940f3a77a56aab32725eb 2024-11-08T13:35:39,950 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 57f3eafc2fac4f1a98250e7c97e05003 2024-11-08T13:35:39,962 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/table/57f3eafc2fac4f1a98250e7c97e05003 is 80, key is testBlockEvictionOnGracefulStop1731072938187/table:state/1731072938565/Put/seqid=0 2024-11-08T13:35:39,963 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=152 B at sequenceid=11 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/table/57f3eafc2fac4f1a98250e7c97e05003 2024-11-08T13:35:39,965 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/6cc437e82d074fe2aa6fa7083afd5a7b as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/6cc437e82d074fe2aa6fa7083afd5a7b 2024-11-08T13:35:39,966 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/6cc437e82d074fe2aa6fa7083afd5a7b, delay=996 ms 2024-11-08T13:35:39,967 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/6cc437e82d074fe2aa6fa7083afd5a7b, entries=10, sequenceid=11, filesize=7.0 K 2024-11-08T13:35:39,967 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/ns/96ab2eced20940f3a77a56aab32725eb as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/96ab2eced20940f3a77a56aab32725eb 2024-11-08T13:35:39,968 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/96ab2eced20940f3a77a56aab32725eb, delay=959 ms 2024-11-08T13:35:39,969 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/96ab2eced20940f3a77a56aab32725eb, entries=2, sequenceid=11, filesize=5.0 K 2024-11-08T13:35:39,969 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/table/57f3eafc2fac4f1a98250e7c97e05003 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/57f3eafc2fac4f1a98250e7c97e05003 2024-11-08T13:35:39,970 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/57f3eafc2fac4f1a98250e7c97e05003, delay=928 ms 2024-11-08T13:35:39,971 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/57f3eafc2fac4f1a98250e7c97e05003, entries=2, sequenceid=11, filesize=5.3 K 2024-11-08T13:35:39,972 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=11, compaction requested=false 2024-11-08T13:35:39,972 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/6cc437e82d074fe2aa6fa7083afd5a7b 2024-11-08T13:35:39,973 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/96ab2eced20940f3a77a56aab32725eb 2024-11-08T13:35:39,973 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/57f3eafc2fac4f1a98250e7c97e05003 2024-11-08T13:35:39,977 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-08T13:35:39,977 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:35:39,977 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:39,978 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731072939896Running coprocessor pre-close hooks at 1731072939896Disabling compacts and flushes for region at 1731072939896Disabling writes for close at 1731072939897 (+1 ms)Obtaining lock to block concurrent updates at 1731072939897Preparing flush snapshotting stores in 1588230740 at 1731072939897Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1797, getHeapSize=3808, getOffHeapSize=0, getCellsCount=14 at 1731072939897Flushing stores of hbase:meta,,1.1588230740 at 1731072939898 (+1 ms)Flushing 1588230740/info: creating writer at 1731072939898Flushing 1588230740/info: appending metadata at 1731072939923 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731072939923Flushing 1588230740/ns: creating writer at 1731072939926 (+3 ms)Flushing 1588230740/ns: appending metadata at 1731072939942 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1731072939942Flushing 1588230740/table: creating writer at 1731072939945 (+3 ms)Flushing 1588230740/table: appending metadata at 1731072939962 (+17 ms)Flushing 1588230740/table: closing flushed file at 1731072939962Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@135659a6: reopening flushed file at 1731072939965 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1279a317: reopening flushed file at 1731072939967 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72a94358: reopening flushed file at 1731072939969 (+2 ms)Finished flush of dataSize ~1.75 KB/1797, heapSize ~3.48 KB/3568, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=11, compaction requested=false at 1731072939972 (+3 ms)Writing region close event to WAL at 1731072939974 (+2 ms)Running coprocessor post-close hooks at 1731072939977 (+3 ms)Closed at 1731072939977 2024-11-08T13:35:39,978 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:40,097 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,33005,1731072937248; all regions closed. 2024-11-08T13:35:40,100 DEBUG [RS:1;6ccfea20c990:33005 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:40,100 INFO [RS:1;6ccfea20c990:33005 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C33005%2C1731072937248.meta:.meta(num 1731072938090) 2024-11-08T13:35:40,103 DEBUG [RS:1;6ccfea20c990:33005 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:40,103 INFO [RS:1;6ccfea20c990:33005 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C33005%2C1731072937248:(num 1731072937668) 2024-11-08T13:35:40,103 DEBUG [RS:1;6ccfea20c990:33005 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:40,103 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:40,103 INFO [RS:1;6ccfea20c990:33005 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:35:40,104 INFO [RS:1;6ccfea20c990:33005 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T13:35:40,104 INFO [RS:1;6ccfea20c990:33005 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:35:40,104 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:35:40,104 INFO [RS:1;6ccfea20c990:33005 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33005 2024-11-08T13:35:40,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,33005,1731072937248 2024-11-08T13:35:40,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:40,109 INFO [RS:1;6ccfea20c990:33005 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:35:40,110 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,33005,1731072937248] 2024-11-08T13:35:40,111 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,33005,1731072937248 already deleted, retry=false 2024-11-08T13:35:40,112 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 6ccfea20c990,33005,1731072937248 on 6ccfea20c990,41401,1731072937165 2024-11-08T13:35:40,115 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 6ccfea20c990,33005,1731072937248, splitWal=true, meta=true 2024-11-08T13:35:40,117 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=7 for 6ccfea20c990,33005,1731072937248 (carryingMeta=true) 6ccfea20c990,33005,1731072937248/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1b9e7fb0[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-08T13:35:40,117 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=7, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 6ccfea20c990,33005,1731072937248, splitWal=true, meta=true 2024-11-08T13:35:40,118 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,33005,1731072937248, splitWal=true, meta=true, isMeta: true 2024-11-08T13:35:40,118 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248-splitting 2024-11-08T13:35:40,119 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248-splitting dir is empty, no logs to split. 2024-11-08T13:35:40,119 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6ccfea20c990,33005,1731072937248 WAL count=0, meta=true 2024-11-08T13:35:40,121 INFO [PEWorker-2 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248-splitting dir is empty, no logs to split. 2024-11-08T13:35:40,121 INFO [PEWorker-2 {}] master.SplitWALManager(105): 6ccfea20c990,33005,1731072937248 WAL count=0, meta=true 2024-11-08T13:35:40,121 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 6ccfea20c990,33005,1731072937248 WAL splitting is done? wals=0, meta=true 2024-11-08T13:35:40,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T13:35:40,123 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:35:40,124 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:35:40,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:40,211 INFO [RS:1;6ccfea20c990:33005 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:35:40,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33005-0x1001f8c292e0002, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:40,211 INFO [RS:1;6ccfea20c990:33005 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,33005,1731072937248; zookeeper connection closed. 2024-11-08T13:35:40,211 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2aeb0586 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2aeb0586 2024-11-08T13:35:40,274 DEBUG [6ccfea20c990:41401 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T13:35:40,275 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=6ccfea20c990,36185,1731072937208 2024-11-08T13:35:40,277 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,36185,1731072937208, state=OPENING 2024-11-08T13:35:40,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:40,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:40,279 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T13:35:40,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:40,279 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:40,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,36185,1731072937208}] 2024-11-08T13:35:40,396 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/6ccfea20c990:0 server-side Connection retries=45 2024-11-08T13:35:40,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:40,396 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:40,397 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T13:35:40,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T13:35:40,397 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T13:35:40,397 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T13:35:40,397 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T13:35:40,398 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41509 2024-11-08T13:35:40,399 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41509 connecting to ZooKeeper ensemble=127.0.0.1:51310 2024-11-08T13:35:40,400 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:40,400 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:40,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:415090x0, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T13:35:40,403 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-08T13:35:40,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41509-0x1001f8c292e0003 connected 2024-11-08T13:35:40,403 DEBUG [pool-298-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-08T13:35:40,404 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T13:35:40,404 INFO [Time-limited test {}] bucket.FileIOEngine(99): Allocating cache 400 MB, on the path:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.cache 2024-11-08T13:35:40,404 INFO [Time-limited test {}] bucket.BucketCache(365): Instantiating BucketCache with acceptableFactor: 0.95, minFactor: 0.85, extraFreeFactor: 0.1, singleFactor: 0.25, multiFactor: 0.5, memoryFactor: 0.25, useStrongRef: false 2024-11-08T13:35:40,404 INFO [Time-limited test {}] bucket.BucketCache(479): Starting BucketCachePersisterThread 2024-11-08T13:35:40,404 INFO [Time-limited test {}] bucket.BucketCachePersister(38): BucketCachePersister started with interval: 100 2024-11-08T13:35:40,404 INFO [Thread-206 {}] bucket.BucketCache(1531): Started retrieving bucket cache from file 2024-11-08T13:35:40,405 INFO [Time-limited test {}] bucket.BucketCache(404): Started bucket cache; ioengine=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.cache, capacity=400 MB, blockSize=64 KB, writerThreadNum=3, writerQLen=64, persistencePath=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence, bucketAllocator=org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator 2024-11-08T13:35:40,405 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T13:35:40,405 INFO [Thread-206 {}] bucket.BucketCache(1553): Reading new chunked format of persistence. 2024-11-08T13:35:40,406 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T13:35:40,406 INFO [Thread-206 {}] bucket.BucketCache(1706): Retrieved 0 of chunks with blockCount = 0. 2024-11-08T13:35:40,407 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T13:35:40,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41509 2024-11-08T13:35:40,408 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41509 2024-11-08T13:35:40,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41509 2024-11-08T13:35:40,409 DEBUG [Thread-206 {}] bucket.BucketCache(1570): Updating region size map after retrieving cached file list 2024-11-08T13:35:40,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41509 2024-11-08T13:35:40,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41509 2024-11-08T13:35:40,410 INFO [Thread-206 {}] bucket.BucketAllocator(343): Cache totalSize=418148352, buckets=199, bucket capacity=2101248=(4*525312)=(FEWEST_ITEMS_IN_BUCKET*(largest configured bucketcache size)) 2024-11-08T13:35:40,410 INFO [Thread-206 {}] bucket.BucketCache(1563): Bucket cache retrieved from file successfully with size: 0 2024-11-08T13:35:40,410 INFO [Thread-206 {}] bucket.BucketCache(414): Persistent bucket cache recovery from /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence is complete. 2024-11-08T13:35:40,411 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(746): ClusterId : b1b5bdb3-88eb-4e63-8d69-b0bf7d219f73 2024-11-08T13:35:40,411 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T13:35:40,412 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T13:35:40,412 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T13:35:40,414 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T13:35:40,414 DEBUG [RS:2;6ccfea20c990:41509 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f561f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=6ccfea20c990/172.17.0.2:0 2024-11-08T13:35:40,427 DEBUG [RS:2;6ccfea20c990:41509 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;6ccfea20c990:41509 2024-11-08T13:35:40,427 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T13:35:40,427 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T13:35:40,427 DEBUG [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T13:35:40,428 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(2659): reportForDuty to master=6ccfea20c990,41401,1731072937165 with port=41509, startcode=1731072940396 2024-11-08T13:35:40,428 DEBUG [RS:2;6ccfea20c990:41509 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T13:35:40,430 INFO [HMaster-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53601, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T13:35:40,431 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41401 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 6ccfea20c990,41509,1731072940396 2024-11-08T13:35:40,431 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41401 {}] master.ServerManager(517): Registering regionserver=6ccfea20c990,41509,1731072940396 2024-11-08T13:35:40,432 DEBUG [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a 2024-11-08T13:35:40,433 DEBUG [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=file:/ 2024-11-08T13:35:40,433 DEBUG [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T13:35:40,434 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:40,435 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T13:35:40,435 DEBUG [RS:2;6ccfea20c990:41509 {}] zookeeper.ZKUtil(111): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/6ccfea20c990,41509,1731072940396 2024-11-08T13:35:40,435 WARN [RS:2;6ccfea20c990:41509 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T13:35:40,435 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [6ccfea20c990,41509,1731072940396] 2024-11-08T13:35:40,435 INFO [RS:2;6ccfea20c990:41509 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:40,435 DEBUG [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(1793): logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,41509,1731072940396 2024-11-08T13:35:40,436 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48267, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T13:35:40,438 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T13:35:40,439 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T13:35:40,439 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T13:35:40,440 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T13:35:40,440 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C36185%2C1731072937208.meta, suffix=.meta, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,36185,1731072937208, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:40,443 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T13:35:40,444 INFO [RS:2;6ccfea20c990:41509 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T13:35:40,444 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,444 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T13:35:40,444 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T13:35:40,444 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/6ccfea20c990:0, corePoolSize=2, maxPoolSize=2 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,445 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/6ccfea20c990:0, corePoolSize=1, maxPoolSize=1 2024-11-08T13:35:40,446 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:40,446 DEBUG [RS:2;6ccfea20c990:41509 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/6ccfea20c990:0, corePoolSize=3, maxPoolSize=3 2024-11-08T13:35:40,449 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,450 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,450 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,450 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,450 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,450 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41509,1731072940396-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T13:35:40,464 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T13:35:40,465 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=6ccfea20c990,41509,1731072940396-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,465 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,465 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.Replication(171): 6ccfea20c990,41509,1731072940396 started 2024-11-08T13:35:40,466 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,36185,1731072937208/6ccfea20c990%2C36185%2C1731072937208.meta.1731072940441.meta 2024-11-08T13:35:40,466 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:40,466 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:40,467 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T13:35:40,467 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T13:35:40,467 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T13:35:40,467 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T13:35:40,467 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:40,467 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T13:35:40,467 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T13:35:40,468 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family info of region 1588230740 2024-11-08T13:35:40,469 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T13:35:40,469 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:40,471 DEBUG [StoreFileOpener-1588230740-info-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/6cc437e82d074fe2aa6fa7083afd5a7b, delay=911 ms 2024-11-08T13:35:40,473 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/6cc437e82d074fe2aa6fa7083afd5a7b 2024-11-08T13:35:40,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:40,473 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family ns of region 1588230740 2024-11-08T13:35:40,474 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T13:35:40,474 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:40,476 DEBUG [StoreFileOpener-1588230740-ns-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/96ab2eced20940f3a77a56aab32725eb, delay=957 ms 2024-11-08T13:35:40,478 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/96ab2eced20940f3a77a56aab32725eb 2024-11-08T13:35:40,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:40,478 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family rep_barrier of region 1588230740 2024-11-08T13:35:40,479 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T13:35:40,479 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:40,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:40,479 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family table of region 1588230740 2024-11-08T13:35:40,480 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T13:35:40,480 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:40,482 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T13:35:40,482 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(1482): Serving as 6ccfea20c990,41509,1731072940396, RpcServer on 6ccfea20c990/172.17.0.2:41509, sessionid=0x1001f8c292e0003 2024-11-08T13:35:40,482 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T13:35:40,482 DEBUG [StoreFileOpener-1588230740-table-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/57f3eafc2fac4f1a98250e7c97e05003, delay=962 ms 2024-11-08T13:35:40,482 DEBUG [RS:2;6ccfea20c990:41509 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 6ccfea20c990,41509,1731072940396 2024-11-08T13:35:40,482 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,41509,1731072940396' 2024-11-08T13:35:40,482 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T13:35:40,483 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T13:35:40,484 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T13:35:40,484 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T13:35:40,484 DEBUG [RS:2;6ccfea20c990:41509 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 6ccfea20c990,41509,1731072940396 2024-11-08T13:35:40,484 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '6ccfea20c990,41509,1731072940396' 2024-11-08T13:35:40,484 DEBUG [StoreOpener-1588230740-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/57f3eafc2fac4f1a98250e7c97e05003 2024-11-08T13:35:40,484 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T13:35:40,484 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T13:35:40,484 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T13:35:40,484 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T13:35:40,485 DEBUG [RS:2;6ccfea20c990:41509 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T13:35:40,485 INFO [RS:2;6ccfea20c990:41509 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T13:35:40,485 INFO [RS:2;6ccfea20c990:41509 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T13:35:40,485 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:40,486 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740 2024-11-08T13:35:40,486 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T13:35:40,486 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T13:35:40,487 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T13:35:40,488 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T13:35:40,489 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=15; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65366640, jitterRate=-0.02596116065979004}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T13:35:40,489 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T13:35:40,490 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731072940467Writing region info on filesystem at 1731072940467Initializing all the Stores at 1731072940468 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072940468Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072940468Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072940468Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731072940468Cleaning up temporary data from old regions at 1731072940486 (+18 ms)Running coprocessor post-open hooks at 1731072940489 (+3 ms)Region opened successfully at 1731072940490 (+1 ms) 2024-11-08T13:35:40,491 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=9, masterSystemTime=1731072940434 2024-11-08T13:35:40,494 DEBUG [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T13:35:40,494 INFO [RS_OPEN_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_META, pid=9}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T13:35:40,494 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=15, regionLocation=6ccfea20c990,36185,1731072937208 2024-11-08T13:35:40,495 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 6ccfea20c990,36185,1731072937208, state=OPEN 2024-11-08T13:35:40,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:40,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:40,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T13:35:40,497 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=9, ppid=8, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=6ccfea20c990,36185,1731072937208 2024-11-08T13:35:40,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:40,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:40,497 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T13:35:40,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-08T13:35:40,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=6ccfea20c990,36185,1731072937208 in 218 msec 2024-11-08T13:35:40,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T13:35:40,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 377 msec 2024-11-08T13:35:40,501 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(207): 6ccfea20c990,33005,1731072937248 had 2 regions 2024-11-08T13:35:40,502 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=7, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 6ccfea20c990,33005,1731072937248, splitWal=true, meta=true, isMeta: false 2024-11-08T13:35:40,503 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248-splitting dir is empty, no logs to split. 2024-11-08T13:35:40,503 INFO [PEWorker-3 {}] master.SplitWALManager(105): 6ccfea20c990,33005,1731072937248 WAL count=0, meta=false 2024-11-08T13:35:40,504 INFO [PEWorker-3 {}] master.SplitLogManager(171): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,33005,1731072937248-splitting dir is empty, no logs to split. 2024-11-08T13:35:40,504 INFO [PEWorker-3 {}] master.SplitWALManager(105): 6ccfea20c990,33005,1731072937248 WAL count=0, meta=false 2024-11-08T13:35:40,504 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 6ccfea20c990,33005,1731072937248 WAL splitting is done? wals=0, meta=false 2024-11-08T13:35:40,506 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN}] 2024-11-08T13:35:40,507 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN 2024-11-08T13:35:40,508 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-08T13:35:40,586 INFO [RS:2;6ccfea20c990:41509 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T13:35:40,588 INFO [RS:2;6ccfea20c990:41509 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=64 MB, rollsize=32 MB, prefix=6ccfea20c990%2C41509%2C1731072940396, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,41509,1731072940396, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs, maxLogs=55 2024-11-08T13:35:40,607 INFO [RS:2;6ccfea20c990:41509 {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/WALs/6ccfea20c990,41509,1731072940396/6ccfea20c990%2C41509%2C1731072940396.1731072940589 2024-11-08T13:35:40,607 DEBUG [RS:2;6ccfea20c990:41509 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-08T13:35:40,658 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(204): Hosts are {6ccfea20c990=0} racks are {/default-rack=0} 2024-11-08T13:35:40,659 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-08T13:35:40,659 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-08T13:35:40,659 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-08T13:35:40,659 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-08T13:35:40,659 INFO [6ccfea20c990:41401 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-08T13:35:40,659 INFO [6ccfea20c990:41401 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-08T13:35:40,659 DEBUG [6ccfea20c990:41401 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-08T13:35:40,659 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=dcb92de9a01da820456a423aff5a6cc4, regionState=OPENING, regionLocation=6ccfea20c990,36185,1731072937208 2024-11-08T13:35:40,660 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 6ccfea20c990:33005 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:33005 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T13:35:40,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,33005,1731072937248, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,33005,1731072937248, seqNum=-1, error=java.net.ConnectException: Call to address=6ccfea20c990:33005 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:33005 2024-11-08T13:35:40,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,33005,1731072937248, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-08T13:35:40,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,33005,1731072937248, seqNum=-1 from cache 2024-11-08T13:35:40,664 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.FailedServers(52): Added failed server with address 6ccfea20c990:33005 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 6ccfea20c990/172.17.0.2:33005 2024-11-08T13:35:40,770 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T13:35:40,770 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=6ccfea20c990,36185,1731072937208, seqNum=-1] 2024-11-08T13:35:40,770 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T13:35:40,773 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44725, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T13:35:40,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN because future has completed 2024-11-08T13:35:40,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcb92de9a01da820456a423aff5a6cc4, server=6ccfea20c990,36185,1731072937208}] 2024-11-08T13:35:40,932 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(132): Open testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:40,932 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7752): Opening region: {ENCODED => dcb92de9a01da820456a423aff5a6cc4, NAME => 'testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.', STARTKEY => '', ENDKEY => ''} 2024-11-08T13:35:40,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testBlockEvictionOnGracefulStop1731072938187 dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(898): Instantiated testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T13:35:40,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7794): checking encryption for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,933 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(7797): checking classloading for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,934 INFO [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=true, cacheDataCompressed=false, prefetchOnOpen=true, for column family family of region dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,934 INFO [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcb92de9a01da820456a423aff5a6cc4 columnFamilyName family 2024-11-08T13:35:40,934 DEBUG [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T13:35:40,936 DEBUG [StoreFileOpener-dcb92de9a01da820456a423aff5a6cc4-family-1 {}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87, delay=985 ms 2024-11-08T13:35:40,937 DEBUG [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] regionserver.StoreEngine(278): loaded file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87 2024-11-08T13:35:40,937 INFO [StoreOpener-dcb92de9a01da820456a423aff5a6cc4-1 {}] regionserver.HStore(327): Store=dcb92de9a01da820456a423aff5a6cc4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T13:35:40,937 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1038): replaying wal for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,938 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,938 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,938 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1048): stopping wal replay for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,938 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1060): Cleaning up temporary data for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,939 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1093): writing seq id for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,940 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1114): Opened dcb92de9a01da820456a423aff5a6cc4; next sequenceid=10; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71775490, jitterRate=0.06953814625740051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-08T13:35:40,940 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:40,940 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegion(1006): Region open journal for dcb92de9a01da820456a423aff5a6cc4: Running coprocessor pre-open hook at 1731072940933Writing region info on filesystem at 1731072940933Initializing all the Stores at 1731072940933Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731072940933Cleaning up temporary data from old regions at 1731072940938 (+5 ms)Running coprocessor post-open hooks at 1731072940940 (+2 ms)Region opened successfully at 1731072940940 2024-11-08T13:35:40,941 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2236): Post open deploy tasks for testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4., pid=11, masterSystemTime=1731072940929 2024-11-08T13:35:40,944 DEBUG [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] regionserver.HRegionServer(2266): Finished post open deploy task for testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:40,944 INFO [RS_OPEN_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_OPEN_REGION, pid=11}] handler.AssignRegionHandler(153): Opened testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:40,944 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=dcb92de9a01da820456a423aff5a6cc4, regionState=OPEN, openSeqNum=10, regionLocation=6ccfea20c990,36185,1731072937208 2024-11-08T13:35:40,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure dcb92de9a01da820456a423aff5a6cc4, server=6ccfea20c990,36185,1731072937208 because future has completed 2024-11-08T13:35:40,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-08T13:35:40,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure dcb92de9a01da820456a423aff5a6cc4, server=6ccfea20c990,36185,1731072937208 in 171 msec 2024-11-08T13:35:40,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=7 2024-11-08T13:35:40,952 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(291): removed crashed server 6ccfea20c990,33005,1731072937248 after splitting done 2024-11-08T13:35:40,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testBlockEvictionOnGracefulStop1731072938187, region=dcb92de9a01da820456a423aff5a6cc4, ASSIGN in 443 msec 2024-11-08T13:35:40,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; ServerCrashProcedure 6ccfea20c990,33005,1731072937248, splitWal=true, meta=true in 840 msec 2024-11-08T13:35:40,982 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T13:35:40,983 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:35:40,983 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:40,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:40,983 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:40,983 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T13:35:40,983 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T13:35:40,983 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=308247435, stopped=false 2024-11-08T13:35:40,983 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=6ccfea20c990,41401,1731072937165 2024-11-08T13:35:40,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:40,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:40,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T13:35:40,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:40,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:40,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:40,985 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:35:40,985 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T13:35:40,985 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.io.hfile.TestBlockEvictionOnRegionMovement.tearDown(TestBlockEvictionOnRegionMovement.java:174) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:40,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:40,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:40,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:40,985 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,36185,1731072937208' ***** 2024-11-08T13:35:40,986 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T13:35:40,986 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T13:35:40,986 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '6ccfea20c990,41509,1731072940396' ***** 2024-11-08T13:35:40,986 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T13:35:40,986 INFO [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T13:35:40,986 INFO [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1739): Disabling cache 2024-11-08T13:35:40,986 INFO [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence 2024-11-08T13:35:40,986 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T13:35:40,986 INFO [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1763): Shutdown bucket cache: IO persistent=true; path to write=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/27554775-3432-897e-40ce-f29aa148dcd2/bucket.persistence 2024-11-08T13:35:40,986 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T13:35:40,986 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T13:35:40,986 INFO [Time-limited test-BucketCacheWriter-1 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-1 exiting, cacheEnabled=false 2024-11-08T13:35:40,986 INFO [Time-limited test-BucketCacheWriter-2 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-2 exiting, cacheEnabled=false 2024-11-08T13:35:40,986 INFO [Time-limited test-BucketCacheWriter-0 {}] bucket.BucketCache$WriterThread(1272): Time-limited test-BucketCacheWriter-0 exiting, cacheEnabled=false 2024-11-08T13:35:40,986 INFO [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T13:35:40,987 INFO [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1769): Shutting down cache persister thread. 2024-11-08T13:35:40,987 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T13:35:40,987 WARN [bucket-cache-persister {}] bucket.BucketCachePersister(64): Interrupting BucketCachePersister thread. java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.io.hfile.bucket.BucketCachePersister.run(BucketCachePersister.java:45) ~[classes/:?] 2024-11-08T13:35:40,997 DEBUG [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1488): Thread RS:0;6ccfea20c990:36185 started persisting bucket cache to file 2024-11-08T13:35:40,997 DEBUG [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1488): Thread RS:2;6ccfea20c990:41509 started persisting bucket cache to file 2024-11-08T13:35:40,997 DEBUG [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:40,997 DEBUG [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1495): Persist in new chunked persistence format. 2024-11-08T13:35:40,997 DEBUG [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:40,997 DEBUG [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1674): persistToFile: before persisting backing map size: 0, fullycachedFiles size: 0, chunkSize: 10000 2024-11-08T13:35:40,999 DEBUG [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T13:35:40,999 DEBUG [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1681): persistToFile: after persisting backing map size: 0, fullycachedFiles size: 0 2024-11-08T13:35:41,000 DEBUG [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731072940997 2024-11-08T13:35:41,000 DEBUG [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1499): PersistToFile: after persisting backing map size: 0, fullycachedFiles size: 0, file name: bucket.persistence1731072940997 2024-11-08T13:35:41,000 DEBUG [RS:2;6ccfea20c990:41509 {}] bucket.BucketCache(1510): Thread RS:2;6ccfea20c990:41509 finished persisting bucket cache to file, renaming 2024-11-08T13:35:41,000 DEBUG [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1510): Thread RS:0;6ccfea20c990:36185 finished persisting bucket cache to file, renaming 2024-11-08T13:35:41,000 WARN [RS:0;6ccfea20c990:36185 {}] bucket.BucketCache(1513): Failed to commit cache persistent file. We might lose cached blocks if RS crashes/restarts before we successfully checkpoint again. 2024-11-08T13:35:41,000 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:35:41,000 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T13:35:41,000 INFO [RS:0;6ccfea20c990:36185 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:35:41,000 INFO [RS:2;6ccfea20c990:41509 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T13:35:41,000 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:35:41,000 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T13:35:41,000 INFO [RS:0;6ccfea20c990:36185 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:35:41,000 INFO [RS:2;6ccfea20c990:41509 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T13:35:41,000 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,41509,1731072940396 2024-11-08T13:35:41,000 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(3091): Received CLOSE for dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:41,000 INFO [RS:2;6ccfea20c990:41509 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:35:41,000 INFO [RS:2;6ccfea20c990:41509 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;6ccfea20c990:41509. 2024-11-08T13:35:41,000 DEBUG [RS:2;6ccfea20c990:41509 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:41,000 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(959): stopping server 6ccfea20c990,36185,1731072937208 2024-11-08T13:35:41,000 DEBUG [RS:2;6ccfea20c990:41509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:41,000 INFO [RS:0;6ccfea20c990:36185 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:35:41,000 INFO [RS:0;6ccfea20c990:36185 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;6ccfea20c990:36185. 2024-11-08T13:35:41,000 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,41509,1731072940396; all regions closed. 2024-11-08T13:35:41,000 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1662): Closing region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T13:35:41,001 DEBUG [RS:0;6ccfea20c990:36185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T13:35:41,001 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dcb92de9a01da820456a423aff5a6cc4, disabling compactions & flushes 2024-11-08T13:35:41,001 DEBUG [RS:0;6ccfea20c990:36185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:41,001 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:41,001 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:41,001 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:35:41,001 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. after waiting 0 ms 2024-11-08T13:35:41,001 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:35:41,001 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:35:41,001 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:41,001 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T13:35:41,002 DEBUG [RS:2;6ccfea20c990:41509 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:41,002 INFO [RS:2;6ccfea20c990:41509 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C41509%2C1731072940396:(num 1731072940589) 2024-11-08T13:35:41,002 DEBUG [RS:2;6ccfea20c990:41509 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:41,002 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:41,002 INFO [RS:2;6ccfea20c990:41509 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:35:41,002 INFO [RS:2;6ccfea20c990:41509 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T13:35:41,002 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T13:35:41,002 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:35:41,003 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T13:35:41,003 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T13:35:41,003 INFO [RS:2;6ccfea20c990:41509 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:35:41,003 INFO [RS:2;6ccfea20c990:41509 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41509 2024-11-08T13:35:41,003 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-08T13:35:41,003 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, dcb92de9a01da820456a423aff5a6cc4=testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4.} 2024-11-08T13:35:41,003 DEBUG [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, dcb92de9a01da820456a423aff5a6cc4 2024-11-08T13:35:41,003 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1662): Closing region hbase:meta,,1.1588230740 during a graceful stop, and cache persistence is on, so setting evict on close to false. 2024-11-08T13:35:41,003 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T13:35:41,003 DEBUG [StoreFileCloser-dcb92de9a01da820456a423aff5a6cc4-family-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/family/6c5753710d524e958fe66e44c82dfd87 2024-11-08T13:35:41,004 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T13:35:41,004 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T13:35:41,004 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T13:35:41,004 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T13:35:41,004 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.21 KB heapSize=2.80 KB 2024-11-08T13:35:41,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,41509,1731072940396 2024-11-08T13:35:41,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:41,005 INFO [RS:2;6ccfea20c990:41509 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:35:41,005 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,41509,1731072940396] 2024-11-08T13:35:41,007 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,41509,1731072940396 already deleted, retry=false 2024-11-08T13:35:41,007 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ccfea20c990,41509,1731072940396 expired; onlineServers=1 2024-11-08T13:35:41,009 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/default/testBlockEvictionOnGracefulStop1731072938187/dcb92de9a01da820456a423aff5a6cc4/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=9 2024-11-08T13:35:41,009 INFO [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:41,009 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dcb92de9a01da820456a423aff5a6cc4: Waiting for close lock at 1731072941000Running coprocessor pre-close hooks at 1731072941001 (+1 ms)Disabling compacts and flushes for region at 1731072941001Disabling writes for close at 1731072941001Writing region close event to WAL at 1731072941004 (+3 ms)Running coprocessor post-close hooks at 1731072941009 (+5 ms)Closed at 1731072941009 2024-11-08T13:35:41,009 DEBUG [RS_CLOSE_REGION-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4. 2024-11-08T13:35:41,013 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFile$WriterFactory(308): Unable to set drop behind on a8cd593700074468bee803483bddc723 2024-11-08T13:35:41,026 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/a8cd593700074468bee803483bddc723 is 209, key is testBlockEvictionOnGracefulStop1731072938187,,1731072938194.dcb92de9a01da820456a423aff5a6cc4./info:regioninfo/1731072940944/Put/seqid=0 2024-11-08T13:35:41,026 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.21 KB at sequenceid=19 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/a8cd593700074468bee803483bddc723 2024-11-08T13:35:41,028 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/.tmp/info/a8cd593700074468bee803483bddc723 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/a8cd593700074468bee803483bddc723 2024-11-08T13:35:41,029 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] hfile.PrefetchExecutor(102): Prefetch requested for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/a8cd593700074468bee803483bddc723, delay=933 ms 2024-11-08T13:35:41,030 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:41,030 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/a8cd593700074468bee803483bddc723, entries=8, sequenceid=19, filesize=6.6 K 2024-11-08T13:35:41,031 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false 2024-11-08T13:35:41,032 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/6cc437e82d074fe2aa6fa7083afd5a7b 2024-11-08T13:35:41,032 DEBUG [StoreFileCloser-1588230740-info-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/info/a8cd593700074468bee803483bddc723 2024-11-08T13:35:41,033 DEBUG [StoreFileCloser-1588230740-ns-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/ns/96ab2eced20940f3a77a56aab32725eb 2024-11-08T13:35:41,033 DEBUG [StoreFileCloser-1588230740-table-1 {}] hfile.PrefetchExecutor(132): Prefetch cancelled for file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/table/57f3eafc2fac4f1a98250e7c97e05003 2024-11-08T13:35:41,037 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/data/hbase/meta/1588230740/recovered.edits/22.seqid, newMaxSeqId=22, maxSeqId=14 2024-11-08T13:35:41,037 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T13:35:41,038 INFO [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:41,038 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731072941003Running coprocessor pre-close hooks at 1731072941003Disabling compacts and flushes for region at 1731072941003Disabling writes for close at 1731072941004 (+1 ms)Obtaining lock to block concurrent updates at 1731072941004Preparing flush snapshotting stores in 1588230740 at 1731072941004Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1238, getHeapSize=2800, getOffHeapSize=0, getCellsCount=8 at 1731072941005 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731072941005Flushing 1588230740/info: creating writer at 1731072941005Flushing 1588230740/info: appending metadata at 1731072941025 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731072941025Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2a0c5de5: reopening flushed file at 1731072941028 (+3 ms)Finished flush of dataSize ~1.21 KB/1238, heapSize ~2.03 KB/2080, currentSize=0 B/0 for 1588230740 in 27ms, sequenceid=19, compaction requested=false at 1731072941031 (+3 ms)Writing region close event to WAL at 1731072941034 (+3 ms)Running coprocessor post-close hooks at 1731072941037 (+3 ms)Closed at 1731072941037 2024-11-08T13:35:41,038 DEBUG [RS_CLOSE_META-regionserver/6ccfea20c990:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T13:35:41,051 INFO [regionserver/6ccfea20c990:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:41,106 INFO [RS:2;6ccfea20c990:41509 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:35:41,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:41,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41509-0x1001f8c292e0003, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:41,106 INFO [RS:2;6ccfea20c990:41509 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,41509,1731072940396; zookeeper connection closed. 2024-11-08T13:35:41,107 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2b71c816 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2b71c816 2024-11-08T13:35:41,203 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(976): stopping server 6ccfea20c990,36185,1731072937208; all regions closed. 2024-11-08T13:35:41,207 DEBUG [RS:0;6ccfea20c990:36185 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:41,207 INFO [RS:0;6ccfea20c990:36185 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C36185%2C1731072937208.meta:.meta(num 1731072940441) 2024-11-08T13:35:41,211 DEBUG [RS:0;6ccfea20c990:36185 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/oldWALs 2024-11-08T13:35:41,211 INFO [RS:0;6ccfea20c990:36185 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 6ccfea20c990%2C36185%2C1731072937208:(num 1731072937669) 2024-11-08T13:35:41,211 DEBUG [RS:0;6ccfea20c990:36185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T13:35:41,211 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T13:35:41,212 INFO [RS:0;6ccfea20c990:36185 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:35:41,212 INFO [RS:0;6ccfea20c990:36185 {}] hbase.ChoreService(370): Chore service for: regionserver/6ccfea20c990:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T13:35:41,213 INFO [RS:0;6ccfea20c990:36185 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:35:41,213 INFO [regionserver/6ccfea20c990:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:35:41,213 INFO [RS:0;6ccfea20c990:36185 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36185 2024-11-08T13:35:41,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/6ccfea20c990,36185,1731072937208 2024-11-08T13:35:41,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T13:35:41,215 INFO [RS:0;6ccfea20c990:36185 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:35:41,216 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [6ccfea20c990,36185,1731072937208] 2024-11-08T13:35:41,217 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/6ccfea20c990,36185,1731072937208 already deleted, retry=false 2024-11-08T13:35:41,217 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 6ccfea20c990,36185,1731072937208 expired; onlineServers=0 2024-11-08T13:35:41,217 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '6ccfea20c990,41401,1731072937165' ***** 2024-11-08T13:35:41,217 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T13:35:41,217 INFO [M:0;6ccfea20c990:41401 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T13:35:41,217 INFO [M:0;6ccfea20c990:41401 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T13:35:41,217 DEBUG [M:0;6ccfea20c990:41401 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T13:35:41,217 DEBUG [M:0;6ccfea20c990:41401 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T13:35:41,217 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T13:35:41,217 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072937462 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.large.0-1731072937462,5,FailOnTimeoutGroup] 2024-11-08T13:35:41,217 DEBUG [master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072937462 {}] cleaner.HFileCleaner(306): Exit Thread[master/6ccfea20c990:0:becomeActiveMaster-HFileCleaner.small.0-1731072937462,5,FailOnTimeoutGroup] 2024-11-08T13:35:41,218 INFO [M:0;6ccfea20c990:41401 {}] hbase.ChoreService(370): Chore service for: master/6ccfea20c990:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T13:35:41,218 INFO [M:0;6ccfea20c990:41401 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T13:35:41,218 DEBUG [M:0;6ccfea20c990:41401 {}] master.HMaster(1795): Stopping service threads 2024-11-08T13:35:41,218 INFO [M:0;6ccfea20c990:41401 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T13:35:41,218 INFO [M:0;6ccfea20c990:41401 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T13:35:41,218 INFO [M:0;6ccfea20c990:41401 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T13:35:41,218 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T13:35:41,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T13:35:41,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T13:35:41,219 DEBUG [M:0;6ccfea20c990:41401 {}] zookeeper.ZKUtil(347): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T13:35:41,219 WARN [M:0;6ccfea20c990:41401 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T13:35:41,219 INFO [M:0;6ccfea20c990:41401 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/.lastflushedseqids 2024-11-08T13:35:41,224 INFO [M:0;6ccfea20c990:41401 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T13:35:41,224 INFO [M:0;6ccfea20c990:41401 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T13:35:41,225 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T13:35:41,225 INFO [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:41,225 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:41,225 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T13:35:41,225 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:41,225 INFO [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=39.51 KB heapSize=49.91 KB 2024-11-08T13:35:41,234 DEBUG [M:0;6ccfea20c990:41401 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 1da3d2c4428546babc951ea5259adab7 2024-11-08T13:35:41,251 DEBUG [M:0;6ccfea20c990:41401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1da3d2c4428546babc951ea5259adab7 is 82, key is hbase:meta,,1/info:regioninfo/1731072940494/Put/seqid=0 2024-11-08T13:35:41,252 INFO [M:0;6ccfea20c990:41401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1da3d2c4428546babc951ea5259adab7 2024-11-08T13:35:41,258 DEBUG [M:0;6ccfea20c990:41401 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 770e6b122bac41b2a12a1cca167ba3e3 2024-11-08T13:35:41,272 DEBUG [M:0;6ccfea20c990:41401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/770e6b122bac41b2a12a1cca167ba3e3 is 783, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731072938570/Put/seqid=0 2024-11-08T13:35:41,272 INFO [M:0;6ccfea20c990:41401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.28 KB at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/770e6b122bac41b2a12a1cca167ba3e3 2024-11-08T13:35:41,279 DEBUG [M:0;6ccfea20c990:41401 {}] hfile.HFile$WriterFactory(308): Unable to set drop behind on 8c330c5c7df94c84a0973d131cc61b76 2024-11-08T13:35:41,292 DEBUG [M:0;6ccfea20c990:41401 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c330c5c7df94c84a0973d131cc61b76 is 69, key is 6ccfea20c990,36185,1731072937208/rs:state/1731072937504/Put/seqid=0 2024-11-08T13:35:41,292 INFO [M:0;6ccfea20c990:41401 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=102 (bloomFilter=true), to=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c330c5c7df94c84a0973d131cc61b76 2024-11-08T13:35:41,294 INFO [M:0;6ccfea20c990:41401 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c330c5c7df94c84a0973d131cc61b76 2024-11-08T13:35:41,294 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1da3d2c4428546babc951ea5259adab7 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1da3d2c4428546babc951ea5259adab7 2024-11-08T13:35:41,296 INFO [M:0;6ccfea20c990:41401 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1da3d2c4428546babc951ea5259adab7, entries=14, sequenceid=102, filesize=5.9 K 2024-11-08T13:35:41,297 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/770e6b122bac41b2a12a1cca167ba3e3 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/770e6b122bac41b2a12a1cca167ba3e3 2024-11-08T13:35:41,298 INFO [M:0;6ccfea20c990:41401 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/770e6b122bac41b2a12a1cca167ba3e3, entries=11, sequenceid=102, filesize=6.5 K 2024-11-08T13:35:41,299 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegionFileSystem(442): Committing file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/8c330c5c7df94c84a0973d131cc61b76 as file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c330c5c7df94c84a0973d131cc61b76 2024-11-08T13:35:41,300 INFO [M:0;6ccfea20c990:41401 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c330c5c7df94c84a0973d131cc61b76 2024-11-08T13:35:41,300 INFO [M:0;6ccfea20c990:41401 {}] regionserver.HStore$StoreFlusherImpl(1990): Added file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/673ed52e-ccd6-3685-37ed-b9a1be9d2950/15de4fa8-5c8e-56c3-d77d-03c5be05638a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/8c330c5c7df94c84a0973d131cc61b76, entries=3, sequenceid=102, filesize=5.3 K 2024-11-08T13:35:41,301 INFO [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(3140): Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=102, compaction requested=false 2024-11-08T13:35:41,303 INFO [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T13:35:41,303 DEBUG [M:0;6ccfea20c990:41401 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731072941224Disabling compacts and flushes for region at 1731072941224Disabling writes for close at 1731072941225 (+1 ms)Obtaining lock to block concurrent updates at 1731072941225Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731072941225Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=40460, getHeapSize=51048, getOffHeapSize=0, getCellsCount=127 at 1731072941225Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731072941226 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731072941226Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731072941251 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731072941251Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731072941253 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731072941271 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731072941271Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731072941274 (+3 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731072941291 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731072941291Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@567be17d: reopening flushed file at 1731072941294 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4a1b3388: reopening flushed file at 1731072941297 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f739297: reopening flushed file at 1731072941299 (+2 ms)Finished flush of dataSize ~39.51 KB/40460, heapSize ~49.62 KB/50808, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 76ms, sequenceid=102, compaction requested=false at 1731072941301 (+2 ms)Writing region close event to WAL at 1731072941303 (+2 ms)Closed at 1731072941303 2024-11-08T13:35:41,304 INFO [M:0;6ccfea20c990:41401 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T13:35:41,304 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T13:35:41,305 INFO [M:0;6ccfea20c990:41401 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41401 2024-11-08T13:35:41,305 INFO [M:0;6ccfea20c990:41401 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T13:35:41,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:41,316 INFO [RS:0;6ccfea20c990:36185 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:35:41,316 INFO [RS:0;6ccfea20c990:36185 {}] regionserver.HRegionServer(1031): Exiting; stopping=6ccfea20c990,36185,1731072937208; zookeeper connection closed. 2024-11-08T13:35:41,316 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36185-0x1001f8c292e0001, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:41,317 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1a12214d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1a12214d 2024-11-08T13:35:41,317 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-08T13:35:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:41,406 INFO [M:0;6ccfea20c990:41401 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T13:35:41,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41401-0x1001f8c292e0000, quorum=127.0.0.1:51310, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T13:35:41,409 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T13:35:41,411 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-08T13:35:41,411 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T13:35:41,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: io.hfile.TestBlockEvictionOnRegionMovement#testBlockEvictionOnGracefulStop Thread=69 (was 49) Potentially hanging thread: HMaster-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-5-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=382 (was 346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 234), ProcessCount=11 (was 11), AvailableMemoryMB=3280 (was 3403)